[llvm-branch-commits] [llvm] AMDGPU/GlobalISel: Switch RegBankLegalize lowering to extended LLTs (PR #208270)
Petar Avramovic via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 10 04:53:08 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/208270
>From 345c3067de99f9b8e02ed2e7da7f98e621410ad9 Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Wed, 8 Jul 2026 17:42:00 +0200
Subject: [PATCH] AMDGPU/GlobalISel: Switch RegBankLegalize lowering to
extended LLTs
Stop using LLT::scalar for lowering in AMDGPURegBankLegalizeHelper
Affects integer lowering code for RegBankLLTMappingApplyID and
custom cpp for LoweringMethodID.
---
.../AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 382 +-
.../AMDGPU/AMDGPURegBankLegalizeHelper.h | 11 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll | 40 +-
.../CodeGen/AMDGPU/GlobalISel/add.v2i16.ll | 122 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll | 104 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll | 210 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 133 +-
.../AMDGPU/GlobalISel/cvt_f32_ubyte.ll | 56 +-
...vergence-divergent-i1-used-outside-loop.ll | 218 +-
.../GlobalISel/divergence-structurizer.ll | 226 +-
.../divergence-temporal-divergent-i1.ll | 173 +-
.../GlobalISel/dynamic-alloca-uniform.ll | 138 +-
.../AMDGPU/GlobalISel/extractelement.i8.ll | 1451 ++----
.../AMDGPU/GlobalISel/extractelement.ll | 80 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll | 14 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll | 19 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll | 63 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll | 19 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll | 14 +-
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 20 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll | 292 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 2511 ++++++----
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 1229 +++--
llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll | 21 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/icmp.ll | 4 +-
.../AMDGPU/GlobalISel/insertelement.i8.ll | 28 +-
.../AMDGPU/GlobalISel/insertelement.ll | 1437 +++---
.../CodeGen/AMDGPU/GlobalISel/llvm.abs.ll | 70 +-
.../GlobalISel/llvm.amdgcn.interp.inreg.ll | 19 +-
.../llvm.amdgcn.make.buffer.rsrc.ll | 44 +-
...llvm.amdgcn.raw.buffer.store.format.f16.ll | 32 +-
.../llvm.amdgcn.raw.ptr.buffer.atomic.add.ll | 32 +-
...vm.amdgcn.raw.ptr.buffer.atomic.cmpswap.ll | 64 +-
.../llvm.amdgcn.raw.ptr.buffer.atomic.fadd.ll | 64 +-
.../llvm.amdgcn.raw.ptr.buffer.atomic.sub.ll | 32 +-
.../llvm.amdgcn.raw.ptr.buffer.atomic.swap.ll | 32 +-
...m.amdgcn.raw.ptr.buffer.load.format.f16.ll | 32 +-
.../llvm.amdgcn.raw.ptr.buffer.load.format.ll | 16 +-
.../llvm.amdgcn.raw.ptr.buffer.load.ll | 192 +-
....amdgcn.raw.ptr.buffer.store.format.f16.ll | 64 +-
....amdgcn.raw.ptr.buffer.store.format.f32.ll | 32 +-
.../llvm.amdgcn.raw.ptr.buffer.store.ll | 80 +-
.../llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll | 32 +-
.../llvm.amdgcn.raw.ptr.tbuffer.load.ll | 16 +-
.../llvm.amdgcn.raw.ptr.tbuffer.store.i8.ll | 96 +-
.../llvm.amdgcn.raw.ptr.tbuffer.store.ll | 80 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll | 3 +-
...vm.amdgcn.struct.buffer.load.format.f16.ll | 12 +-
...lvm.amdgcn.struct.ptr.buffer.atomic.add.ll | 32 +-
...amdgcn.struct.ptr.buffer.atomic.cmpswap.ll | 64 +-
...vm.amdgcn.struct.ptr.buffer.atomic.fadd.ll | 64 +-
...lvm.amdgcn.struct.ptr.buffer.atomic.sub.ll | 32 +-
...vm.amdgcn.struct.ptr.buffer.atomic.swap.ll | 32 +-
...mdgcn.struct.ptr.buffer.load.format.f16.ll | 28 +-
...vm.amdgcn.struct.ptr.buffer.load.format.ll | 16 +-
.../llvm.amdgcn.struct.ptr.buffer.load.ll | 16 +-
...dgcn.struct.ptr.buffer.store.format.f32.ll | 16 +-
.../llvm.amdgcn.struct.ptr.buffer.store.ll | 16 +-
...llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll | 28 +-
.../llvm.amdgcn.struct.ptr.tbuffer.load.ll | 16 +-
.../llvm.amdgcn.struct.tbuffer.load.f16.ll | 12 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll | 3 +-
.../AMDGPU/GlobalISel/load-uniform-in-vgpr.ll | 112 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll | 262 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll | 36 +-
.../AMDGPU/GlobalISel/minmaxabs-i64.ll | 10 +-
.../CodeGen/AMDGPU/GlobalISel/mubuf-global.ll | 261 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 3424 +++++++-------
llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll | 63 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll | 218 +-
.../AMDGPU/GlobalISel/regbankselect-abs.mir | 22 +-
.../AMDGPU/GlobalISel/regbankselect-add.mir | 45 +-
.../GlobalISel/regbankselect-add.s16.mir | 8 +-
.../GlobalISel/regbankselect-add.v2s16.mir | 16 +-
.../regbankselect-amdgcn-s-buffer-load.mir | 47 +-
.../regbankselect-amdgcn.div.fmas.mir | 4 +-
.../regbankselect-amdgcn.image.load.1d.ll | 64 +-
.../regbankselect-amdgcn.image.sample.1d.ll | 104 +-
.../GlobalISel/regbankselect-amdgcn.kill.mir | 12 +-
.../regbankselect-amdgcn.raw.buffer.load.ll | 28 +-
...egbankselect-amdgcn.raw.ptr.buffer.load.ll | 28 +-
.../regbankselect-amdgcn.s.buffer.load.ll | 1146 ++---
...regbankselect-amdgcn.struct.buffer.load.ll | 28 +-
...egbankselect-amdgcn.struct.buffer.store.ll | 28 +-
...ankselect-amdgcn.struct.ptr.buffer.load.ll | 28 +-
...nkselect-amdgcn.struct.ptr.buffer.store.ll | 28 +-
.../regbankselect-amdgcn.wqm.demote.mir | 12 +-
.../regbankselect-amdgpu-ffbh-u32.mir | 10 +-
.../regbankselect-amdgpu-ffbl-b32.mir | 10 +-
.../GlobalISel/regbankselect-and-s1.mir | 26 +-
.../GlobalISel/regbankselect-anyext.mir | 38 +-
.../AMDGPU/GlobalISel/regbankselect-ashr.mir | 28 +-
.../GlobalISel/regbankselect-brcond.mir | 14 +-
.../AMDGPU/GlobalISel/regbankselect-call.ll | 8 +-
.../regbankselect-ctlz-zero-poison.mir | 10 +-
.../AMDGPU/GlobalISel/regbankselect-ctpop.mir | 6 +-
.../regbankselect-cttz-zero-poison.mir | 10 +-
.../regbankselect-dyn-stackalloc.mir | 272 +-
.../regbankselect-extract-vector-elt.mir | 392 +-
.../regbankselect-fcanonicalize.mir | 4 +-
.../AMDGPU/GlobalISel/regbankselect-fcmp.mir | 2 +-
.../AMDGPU/GlobalISel/regbankselect-fexp2.mir | 8 +-
.../GlobalISel/regbankselect-freeze.mir | 4 +-
.../AMDGPU/GlobalISel/regbankselect-fsqrt.mir | 4 +-
.../GlobalISel/regbankselect-icmp.s16.mir | 6 +-
.../AMDGPU/GlobalISel/regbankselect-load.mir | 110 +-
.../AMDGPU/GlobalISel/regbankselect-lshr.mir | 30 +-
.../regbankselect-mui-regbanklegalize.mir | 75 +-
.../AMDGPU/GlobalISel/regbankselect-mui.mir | 75 +-
.../AMDGPU/GlobalISel/regbankselect-mul.mir | 34 +-
.../AMDGPU/GlobalISel/regbankselect-or.mir | 22 +-
.../AMDGPU/GlobalISel/regbankselect-sadde.mir | 18 +-
.../GlobalISel/regbankselect-salu-float.mir | 18 +-
.../AMDGPU/GlobalISel/regbankselect-sbfx.mir | 110 +-
.../GlobalISel/regbankselect-select.mir | 130 +-
.../GlobalISel/regbankselect-sext-inreg.mir | 52 +-
.../AMDGPU/GlobalISel/regbankselect-sext.mir | 93 +-
.../AMDGPU/GlobalISel/regbankselect-shl.mir | 24 +-
.../AMDGPU/GlobalISel/regbankselect-smax.mir | 36 +-
.../AMDGPU/GlobalISel/regbankselect-smin.mir | 36 +-
...gbankselect-split-scalar-load-metadata.mir | 15 +-
.../AMDGPU/GlobalISel/regbankselect-ssube.mir | 18 +-
.../AMDGPU/GlobalISel/regbankselect-sub.mir | 32 +-
.../AMDGPU/GlobalISel/regbankselect-trunc.mir | 22 +-
.../AMDGPU/GlobalISel/regbankselect-uadde.mir | 18 +-
.../AMDGPU/GlobalISel/regbankselect-uaddo.mir | 2 +-
.../AMDGPU/GlobalISel/regbankselect-ubfx.mir | 106 +-
.../AMDGPU/GlobalISel/regbankselect-umax.mir | 38 +-
.../AMDGPU/GlobalISel/regbankselect-umin.mir | 38 +-
.../regbankselect-uniform-load-noclobber.mir | 40 +-
.../AMDGPU/GlobalISel/regbankselect-usube.mir | 18 +-
.../AMDGPU/GlobalISel/regbankselect-usubo.mir | 2 +-
.../regbankselect-waterfall-call.mir | 16 +-
.../regbankselect-widen-scalar-loads.mir | 54 +-
.../AMDGPU/GlobalISel/regbankselect-xor.mir | 26 +-
.../AMDGPU/GlobalISel/regbankselect-zext.mir | 90 +-
.../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 1713 ++++---
llvm/test/CodeGen/AMDGPU/GlobalISel/sbfx.ll | 18 +-
.../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 3505 ++++++++------
.../CodeGen/AMDGPU/GlobalISel/sext_inreg.ll | 174 +-
.../AMDGPU/GlobalISel/shl-ext-reduce.ll | 97 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll | 272 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll | 15 +-
.../CodeGen/AMDGPU/GlobalISel/srem.i32.ll | 22 +-
.../CodeGen/AMDGPU/GlobalISel/srem.i64.ll | 3451 ++++++++------
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 1739 ++++---
.../AMDGPU/GlobalISel/store-local.128.ll | 99 +-
.../AMDGPU/GlobalISel/store-local.96.ll | 77 +-
.../AMDGPU/GlobalISel/strict_fma.f16.ll | 56 +-
.../CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll | 177 +-
.../GlobalISel/true16-merge-values-s16.ll | 64 +-
.../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 228 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ubfx.ll | 6 +-
.../CodeGen/AMDGPU/GlobalISel/udiv.i64.ll | 806 ++--
.../AMDGPU/GlobalISel/unmerge-sgpr-s16.mir | 28 +-
.../CodeGen/AMDGPU/GlobalISel/urem.i64.ll | 782 ++--
.../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 430 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll | 103 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll | 66 +-
llvm/test/CodeGen/AMDGPU/add-max.ll | 31 +-
llvm/test/CodeGen/AMDGPU/bfi_int.ll | 216 +-
...ffer-fat-pointers-contents-legalization.ll | 2 +-
.../CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll | 10 +-
llvm/test/CodeGen/AMDGPU/ctlz.ll | 62 +-
llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll | 40 +-
llvm/test/CodeGen/AMDGPU/ctpop16.ll | 24 +-
llvm/test/CodeGen/AMDGPU/ctpop64.ll | 62 +-
llvm/test/CodeGen/AMDGPU/cttz.ll | 34 +-
llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll | 4 +-
llvm/test/CodeGen/AMDGPU/ds-alignment.ll | 22 +-
.../test/CodeGen/AMDGPU/dynamic_stackalloc.ll | 419 +-
llvm/test/CodeGen/AMDGPU/fadd.f16.ll | 107 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 3481 +++++++++-----
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 89 +-
llvm/test/CodeGen/AMDGPU/fmaxnum.ll | 133 +-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 89 +-
llvm/test/CodeGen/AMDGPU/fminnum.ll | 133 +-
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll | 1027 +++--
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 168 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll | 410 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 628 ++-
llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll | 261 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 500 +-
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 618 +--
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 323 +-
llvm/test/CodeGen/AMDGPU/literal64.ll | 41 +-
.../AMDGPU/llvm.amdgcn.div.fixup.f16.ll | 138 +-
.../AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll | 14 +-
.../AMDGPU/llvm.amdgcn.interp.inreg.ll | 42 +-
.../CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll | 32 +-
llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll | 3743 ++++++++++-----
llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll | 4041 +++++++++++------
llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll | 3717 ++++++++++-----
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 123 +-
llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll | 8 +-
llvm/test/CodeGen/AMDGPU/llvm.modf.ll | 194 +-
llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll | 2173 ++++++---
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 220 +-
llvm/test/CodeGen/AMDGPU/lrint.ll | 594 ++-
llvm/test/CodeGen/AMDGPU/lround.ll | 388 +-
llvm/test/CodeGen/AMDGPU/roundeven.ll | 46 +-
llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll | 128 +-
.../CodeGen/AMDGPU/scale-offset-global.ll | 38 +-
llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll | 35 +-
llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll | 19 +-
llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll | 19 +-
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | 35 +-
llvm/test/CodeGen/AMDGPU/v_pack.ll | 77 +-
llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll | 190 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 2657 +++++++----
llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 551 ++-
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 2021 ++++++---
llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll | 2406 ++++++----
213 files changed, 39895 insertions(+), 25334 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index b9554d7858d32..369734004c3c0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -228,7 +228,7 @@ bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
// Build the comparison(s), CurrentLaneReg == OpReg.
unsigned OpSize = OpTy.getSizeInBits();
unsigned PartSize = (OpSize % 64 == 0) ? 64 : 32;
- LLT PartTy = LLT::scalar(PartSize);
+ LLT PartTy = LLT::integer(PartSize);
unsigned NumParts = OpSize / PartSize;
SmallVector<Register, 8> OpParts;
SmallVector<Register, 8> CurrentLaneParts;
@@ -264,7 +264,7 @@ bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
// Copy vcc to sgpr32/64, ballot becomes a no-op during instruction selection.
Register CondRegLM =
- MRI.createVirtualRegister({WaveRC, LLT::scalar(IsWave32 ? 32 : 64)});
+ MRI.createVirtualRegister({WaveRC, LLT::integer(IsWave32 ? 32 : 64)});
B.buildIntrinsic(Intrinsic::amdgcn_ballot, CondRegLM).addReg(CondReg);
// Update EXEC, save the original EXEC value to SavedExec.
@@ -311,8 +311,8 @@ unsigned RegBankLegalizeHelper::setBufferOffsets(
getIConstantVRegSExtVal(CombinedOffset, MRI)) {
uint32_t SOffset, ImmOffset;
if (TII.splitMUBUFOffset(*Imm, SOffset, ImmOffset, Alignment)) {
- VOffsetReg = B.buildConstant({VgprRB, S32}, 0).getReg(0);
- SOffsetReg = B.buildConstant({SgprRB, S32}, SOffset).getReg(0);
+ VOffsetReg = B.buildConstant(VgprRB_I32, 0).getReg(0);
+ SOffsetReg = B.buildConstant(SgprRB_I32, SOffset).getReg(0);
InstOffsetVal = ImmOffset;
return SOffset + ImmOffset;
}
@@ -326,13 +326,13 @@ unsigned RegBankLegalizeHelper::setBufferOffsets(
TII.splitMUBUFOffset(Offset, SOffset, ImmOffset, Alignment)) {
if (Base.isValid() && MRI.getRegBank(Base) == VgprRB) {
VOffsetReg = Base;
- SOffsetReg = B.buildConstant({SgprRB, S32}, SOffset).getReg(0);
+ SOffsetReg = B.buildConstant(SgprRB_I32, SOffset).getReg(0);
InstOffsetVal = ImmOffset;
return 0;
}
// If we have SGPR base, we can use it for soffset.
if (SOffset == 0) {
- VOffsetReg = B.buildConstant({VgprRB, S32}, 0).getReg(0);
+ VOffsetReg = B.buildConstant(VgprRB_I32, 0).getReg(0);
SOffsetReg = Base;
InstOffsetVal = ImmOffset;
return 0;
@@ -362,9 +362,9 @@ unsigned RegBankLegalizeHelper::setBufferOffsets(
if (MRI.getRegBank(CombinedOffset) == VgprRB) {
VOffsetReg = CombinedOffset;
} else {
- VOffsetReg = B.buildCopy({VgprRB, S32}, CombinedOffset).getReg(0);
+ VOffsetReg = B.buildCopy(VgprRB_I32, CombinedOffset).getReg(0);
}
- SOffsetReg = B.buildConstant({SgprRB, S32}, 0).getReg(0);
+ SOffsetReg = B.buildConstant(SgprRB_I32, 0).getReg(0);
return 0;
}
@@ -378,7 +378,7 @@ bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
Register Base = MI.getOperand(1).getReg();
LLT PtrTy = MRI.getType(Base);
const RegisterBank *PtrRB = MRI.getRegBankOrNull(Base);
- LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits());
+ LLT OffsetTy = LLT::integer(PtrTy.getSizeInBits());
SmallVector<Register, 4> LoadPartRegs;
unsigned ByteOffset = 0;
@@ -459,11 +459,11 @@ bool RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
if (MI.getOpcode() == G_LOAD) {
B.buildLoad(Dst, Ptr, *WideMMO);
} else {
- auto Load = B.buildLoad(SgprRB_S32, Ptr, *WideMMO);
+ auto Load = B.buildLoad(SgprRB_I32, Ptr, *WideMMO);
if (MI.getOpcode() == G_ZEXTLOAD) {
APInt Mask = APInt::getLowBitsSet(S32.getSizeInBits(), MemSize);
- auto MaskCst = B.buildConstant(SgprRB_S32, Mask);
+ auto MaskCst = B.buildConstant(SgprRB_I32, Mask);
B.buildAnd(Dst, Load, MaskCst);
} else {
assert(MI.getOpcode() == G_SEXTLOAD);
@@ -486,9 +486,9 @@ bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
auto False = B.buildConstant({VgprRB, Ty}, 0);
B.buildSelect(Dst, Src, True, False);
} else if (Ty == S64) {
- auto True = B.buildConstant({VgprRB_S32}, TrueExtCst);
- auto False = B.buildConstant({VgprRB_S32}, 0);
- auto Lo = B.buildSelect({VgprRB_S32}, Src, True, False);
+ auto True = B.buildConstant({VgprRB_I32}, TrueExtCst);
+ auto False = B.buildConstant({VgprRB_I32}, 0);
+ auto Lo = B.buildSelect({VgprRB_I32}, Src, True, False);
MachineInstrBuilder Hi;
switch (Opc) {
case G_SEXT:
@@ -498,7 +498,7 @@ bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
Hi = False;
break;
case G_ANYEXT:
- Hi = B.buildUndef({VgprRB_S32});
+ Hi = B.buildUndef({VgprRB_I32});
break;
default:
reportGISelFailure(
@@ -520,32 +520,33 @@ bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
}
std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
- auto PackedS32 = B.buildBitcast(SgprRB_S32, Reg);
- auto Mask = B.buildConstant(SgprRB_S32, 0x0000ffff);
- auto Lo = B.buildAnd(SgprRB_S32, PackedS32, Mask);
- auto Hi = B.buildLShr(SgprRB_S32, PackedS32, B.buildConstant(SgprRB_S32, 16));
+ auto PackedI32 = B.buildBitcast(SgprRB_I32, Reg);
+ auto Mask = B.buildConstant(SgprRB_I32, 0x0000ffff);
+ auto Lo = B.buildAnd(SgprRB_I32, PackedI32, Mask);
+ auto Hi = B.buildLShr(SgprRB_I32, PackedI32, B.buildConstant(SgprRB_I32, 16));
return {Lo.getReg(0), Hi.getReg(0)};
}
std::pair<Register, Register> RegBankLegalizeHelper::unpackSExt(Register Reg) {
- auto PackedS32 = B.buildBitcast(SgprRB_S32, Reg);
- auto Lo = B.buildSExtInReg(SgprRB_S32, PackedS32, 16);
- auto Hi = B.buildAShr(SgprRB_S32, PackedS32, B.buildConstant(SgprRB_S32, 16));
+ auto PackedI32 = B.buildBitcast(SgprRB_I32, Reg);
+ auto Lo = B.buildSExtInReg(SgprRB_I32, PackedI32, 16);
+ auto Hi = B.buildAShr(SgprRB_I32, PackedI32, B.buildConstant(SgprRB_I32, 16));
return {Lo.getReg(0), Hi.getReg(0)};
}
std::pair<Register, Register> RegBankLegalizeHelper::unpackAExt(Register Reg) {
- auto PackedS32 = B.buildBitcast(SgprRB_S32, Reg);
- auto Lo = PackedS32;
- auto Hi = B.buildLShr(SgprRB_S32, PackedS32, B.buildConstant(SgprRB_S32, 16));
+ auto PackedI32 = B.buildBitcast(SgprRB_I32, Reg);
+ auto Lo = PackedI32;
+ auto Hi = B.buildLShr(SgprRB_I32, PackedI32, B.buildConstant(SgprRB_I32, 16));
return {Lo.getReg(0), Hi.getReg(0)};
}
std::pair<Register, Register>
RegBankLegalizeHelper::unpackAExtTruncS16(Register Reg) {
auto [Lo32, Hi32] = unpackAExt(Reg);
- return {B.buildTrunc(SgprRB_S16, Lo32).getReg(0),
- B.buildTrunc(SgprRB_S16, Hi32).getReg(0)};
+ LLT EltTy = MRI.getType(Reg).getElementType();
+ return {B.buildTrunc({SgprRB, EltTy}, Lo32).getReg(0),
+ B.buildTrunc({SgprRB, EltTy}, Hi32).getReg(0)};
}
bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
@@ -554,22 +555,22 @@ bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
case AMDGPU::G_SHL: {
auto [Val0, Val1] = unpackAExt(MI.getOperand(1).getReg());
auto [Amt0, Amt1] = unpackAExt(MI.getOperand(2).getReg());
- Lo = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val0, Amt0}).getReg(0);
- Hi = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val1, Amt1}).getReg(0);
+ Lo = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val0, Amt0}).getReg(0);
+ Hi = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val1, Amt1}).getReg(0);
break;
}
case AMDGPU::G_LSHR: {
auto [Val0, Val1] = unpackZExt(MI.getOperand(1).getReg());
auto [Amt0, Amt1] = unpackZExt(MI.getOperand(2).getReg());
- Lo = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val0, Amt0}).getReg(0);
- Hi = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val1, Amt1}).getReg(0);
+ Lo = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val0, Amt0}).getReg(0);
+ Hi = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val1, Amt1}).getReg(0);
break;
}
case AMDGPU::G_ASHR: {
auto [Val0, Val1] = unpackSExt(MI.getOperand(1).getReg());
auto [Amt0, Amt1] = unpackSExt(MI.getOperand(2).getReg());
- Lo = B.buildAShr(SgprRB_S32, Val0, Amt0).getReg(0);
- Hi = B.buildAShr(SgprRB_S32, Val1, Amt1).getReg(0);
+ Lo = B.buildAShr(SgprRB_I32, Val0, Amt0).getReg(0);
+ Hi = B.buildAShr(SgprRB_I32, Val1, Amt1).getReg(0);
break;
}
default:
@@ -592,9 +593,9 @@ bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
// For signed operations, use sign extension
auto [Val0_Lo, Val0_Hi] = unpackSExt(MI.getOperand(1).getReg());
auto [Val1_Lo, Val1_Hi] = unpackSExt(MI.getOperand(2).getReg());
- Lo = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val0_Lo, Val1_Lo})
+ Lo = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val0_Lo, Val1_Lo})
.getReg(0);
- Hi = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val0_Hi, Val1_Hi})
+ Hi = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val0_Hi, Val1_Hi})
.getReg(0);
break;
}
@@ -603,9 +604,9 @@ bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
// For unsigned operations, use zero extension
auto [Val0_Lo, Val0_Hi] = unpackZExt(MI.getOperand(1).getReg());
auto [Val1_Lo, Val1_Hi] = unpackZExt(MI.getOperand(2).getReg());
- Lo = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val0_Lo, Val1_Lo})
+ Lo = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val0_Lo, Val1_Lo})
.getReg(0);
- Hi = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Val0_Hi, Val1_Hi})
+ Hi = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Val0_Hi, Val1_Hi})
.getReg(0);
break;
}
@@ -623,8 +624,8 @@ bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
bool RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
auto [Op1Lo, Op1Hi] = unpackAExt(MI.getOperand(1).getReg());
auto [Op2Lo, Op2Hi] = unpackAExt(MI.getOperand(2).getReg());
- auto ResLo = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Op1Lo, Op2Lo});
- auto ResHi = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Op1Hi, Op2Hi});
+ auto ResLo = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Op1Lo, Op2Lo});
+ auto ResHi = B.buildInstr(MI.getOpcode(), {SgprRB_I32}, {Op1Hi, Op2Hi});
B.buildBuildVectorTrunc(MI.getOperand(0).getReg(),
{ResLo.getReg(0), ResHi.getReg(0)});
MI.eraseFromParent();
@@ -663,7 +664,7 @@ bool RegBankLegalizeHelper::lowerSBufToBuf(MachineInstr &MI,
// If only the offset is divergent, emit a MUBUF buffer load
// instead. We can assume that the buffer is unswizzled.
Register RSrc = MI.getOperand(1).getReg();
- Register VIndex = B.buildConstant(VgprRB_S32, 0).getReg(0);
+ Register VIndex = B.buildConstant(VgprRB_I32, 0).getReg(0);
unsigned Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
switch (MI.getOpcode()) {
case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
@@ -729,7 +730,7 @@ bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
// Src >> LSBit Hi|Lo: x?????syyyyyyl??? -> xxxx?????syyyyyyl
unsigned SHROpc = Signed ? AMDGPU::G_ASHR : AMDGPU::G_LSHR;
- auto SHRSrc = B.buildInstr(SHROpc, {{VgprRB, S64}}, {Src, LSBit});
+ auto SHRSrc = B.buildInstr(SHROpc, {VgprRB_I64}, {Src, LSBit});
auto ConstWidth = getIConstantVRegValWithLookThrough(Width, MRI);
@@ -737,36 +738,36 @@ bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
// << (64 - Width): Hi|Lo: xxxx?????syyyyyyl -> syyyyyyl000000000
// >> (64 - Width): Hi|Lo: syyyyyyl000000000 -> ssssssssssyyyyyyl
if (!ConstWidth) {
- auto Amt = B.buildSub(VgprRB_S32, B.buildConstant(SgprRB_S32, 64), Width);
- auto SignBit = B.buildShl({VgprRB, S64}, SHRSrc, Amt);
+ auto Amt = B.buildSub(VgprRB_I32, B.buildConstant(SgprRB_I32, 64), Width);
+ auto SignBit = B.buildShl(VgprRB_I64, SHRSrc, Amt);
B.buildInstr(SHROpc, {Dst}, {SignBit, Amt});
MI.eraseFromParent();
return true;
}
uint64_t WidthImm = ConstWidth->Value.getZExtValue();
- auto UnmergeSHRSrc = B.buildUnmerge(VgprRB_S32, SHRSrc);
+ auto UnmergeSHRSrc = B.buildUnmerge(VgprRB_I32, SHRSrc);
Register SHRSrcLo = UnmergeSHRSrc.getReg(0);
Register SHRSrcHi = UnmergeSHRSrc.getReg(1);
- auto Zero = B.buildConstant({VgprRB, S32}, 0);
+ auto Zero = B.buildConstant(VgprRB_I32, 0);
unsigned BFXOpc = Signed ? AMDGPU::G_SBFX : AMDGPU::G_UBFX;
if (WidthImm <= 32) {
// SHRSrc Hi|Lo: ????????|???syyyl -> ????????|ssssyyyl
- auto Lo = B.buildInstr(BFXOpc, {VgprRB_S32}, {SHRSrcLo, Zero, Width});
+ auto Lo = B.buildInstr(BFXOpc, {VgprRB_I32}, {SHRSrcLo, Zero, Width});
MachineInstrBuilder Hi;
if (Signed) {
// SHRSrc Hi|Lo: ????????|ssssyyyl -> ssssssss|ssssyyyl
- Hi = B.buildAShr(VgprRB_S32, Lo, B.buildConstant(VgprRB_S32, 31));
+ Hi = B.buildAShr(VgprRB_I32, Lo, B.buildConstant(VgprRB_I32, 31));
} else {
// SHRSrc Hi|Lo: ????????|000syyyl -> 00000000|000syyyl
Hi = Zero;
}
B.buildMergeLikeInstr(Dst, {Lo, Hi});
} else {
- auto Amt = B.buildConstant(VgprRB_S32, WidthImm - 32);
+ auto Amt = B.buildConstant(VgprRB_I32, WidthImm - 32);
// SHRSrc Hi|Lo: ??????sy|yyyyyyyl -> sssssssy|yyyyyyyl
- auto Hi = B.buildInstr(BFXOpc, {VgprRB_S32}, {SHRSrcHi, Zero, Amt});
+ auto Hi = B.buildInstr(BFXOpc, {VgprRB_I32}, {SHRSrcHi, Zero, Amt});
B.buildMergeLikeInstr(Dst, {SHRSrcLo, Hi});
}
@@ -787,10 +788,10 @@ bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
// field offset in low and size in high 16 bits.
// Src1 Hi16|Lo16 = Size|FieldOffset
- auto Mask = B.buildConstant(SgprRB_S32, maskTrailingOnes<unsigned>(6));
- auto FieldOffset = B.buildAnd(SgprRB_S32, LSBit, Mask);
- auto Size = B.buildShl(SgprRB_S32, Width, B.buildConstant(SgprRB_S32, 16));
- auto Src1 = B.buildOr(SgprRB_S32, FieldOffset, Size);
+ auto Mask = B.buildConstant(SgprRB_I32, maskTrailingOnes<unsigned>(6));
+ auto FieldOffset = B.buildAnd(SgprRB_I32, LSBit, Mask);
+ auto Size = B.buildShl(SgprRB_I32, Width, B.buildConstant(SgprRB_I32, 16));
+ auto Src1 = B.buildOr(SgprRB_I32, FieldOffset, Size);
unsigned Opc32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
unsigned Opc64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
unsigned Opc = Ty == S32 ? Opc32 : Opc64;
@@ -798,7 +799,7 @@ bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
// Select machine instruction, because of reg class constraining, insert
// copies from reg class to reg bank.
auto S_BFE = B.buildInstr(Opc, {{SgprRB, Ty}},
- {B.buildCopy(Ty, Src), B.buildCopy(S32, Src1)});
+ {B.buildCopy(Ty, Src), B.buildCopy(I32, Src1)});
constrainSelectedInstRegOperands(*S_BFE, *ST.getInstrInfo(),
*ST.getRegisterInfo(), RBI);
@@ -811,7 +812,7 @@ bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
Register Dst = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(Dst);
assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64);
- LLT Ty = DstTy == V4S16 ? V2S16 : S32;
+ LLT Ty = DstTy.divide(2);
auto Op1 = B.buildUnmerge({VgprRB, Ty}, MI.getOperand(1).getReg());
auto Op2 = B.buildUnmerge({VgprRB, Ty}, MI.getOperand(2).getReg());
unsigned Opc = MI.getOpcode();
@@ -828,17 +829,17 @@ bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
bool RegBankLegalizeHelper::lowerSplitTo32Mul(MachineInstr &MI) {
Register Dst = MI.getOperand(0).getReg();
assert(MRI.getType(Dst) == S64);
- auto Op1 = B.buildUnmerge({VgprRB_S32}, MI.getOperand(1).getReg());
- auto Op2 = B.buildUnmerge({VgprRB_S32}, MI.getOperand(2).getReg());
+ auto Op1 = B.buildUnmerge({VgprRB_I32}, MI.getOperand(1).getReg());
+ auto Op2 = B.buildUnmerge({VgprRB_I32}, MI.getOperand(2).getReg());
// TODO: G_AMDGPU_MAD_* optimizations for G_MUL divergent S64 operation to
// match GlobalISel with old regbankselect.
- auto Lo = B.buildMul(VgprRB_S32, Op1.getReg(0), Op2.getReg(0));
- auto Carry = B.buildUMulH(VgprRB_S32, Op1.getReg(0), Op2.getReg(0));
- auto MulLo0Hi1 = B.buildMul(VgprRB_S32, Op1.getReg(0), Op2.getReg(1));
- auto MulHi0Lo1 = B.buildMul(VgprRB_S32, Op1.getReg(1), Op2.getReg(0));
- auto Sum = B.buildAdd(VgprRB_S32, MulLo0Hi1, MulHi0Lo1);
- auto Hi = B.buildAdd(VgprRB_S32, Sum, Carry);
+ auto Lo = B.buildMul(VgprRB_I32, Op1.getReg(0), Op2.getReg(0));
+ auto Carry = B.buildUMulH(VgprRB_I32, Op1.getReg(0), Op2.getReg(0));
+ auto MulLo0Hi1 = B.buildMul(VgprRB_I32, Op1.getReg(0), Op2.getReg(1));
+ auto MulHi0Lo1 = B.buildMul(VgprRB_I32, Op1.getReg(1), Op2.getReg(0));
+ auto Sum = B.buildAdd(VgprRB_I32, MulLo0Hi1, MulHi0Lo1);
+ auto Hi = B.buildAdd(VgprRB_I32, Sum, Carry);
B.buildMergeLikeInstr(Dst, {Lo, Hi});
MI.eraseFromParent();
@@ -853,10 +854,11 @@ bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
auto Flags = MI.getFlags();
auto [Op1Lo, Op1Hi] = unpackAExtTruncS16(MI.getOperand(1).getReg());
+ LLT EltTy = MRI.getType(Dst).getElementType();
if (NumOps == 2) {
- auto Lo = B.buildInstr(Opc, {SgprRB_S16}, {Op1Lo}, Flags);
- auto Hi = B.buildInstr(Opc, {SgprRB_S16}, {Op1Hi}, Flags);
+ auto Lo = B.buildInstr(Opc, {{SgprRB, EltTy}}, {Op1Lo}, Flags);
+ auto Hi = B.buildInstr(Opc, {{SgprRB, EltTy}}, {Op1Hi}, Flags);
B.buildMergeLikeInstr(Dst, {Lo, Hi});
MI.eraseFromParent();
return true;
@@ -865,8 +867,8 @@ bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
auto [Op2Lo, Op2Hi] = unpackAExtTruncS16(MI.getOperand(2).getReg());
if (NumOps == 3) {
- auto Lo = B.buildInstr(Opc, {SgprRB_S16}, {Op1Lo, Op2Lo}, Flags);
- auto Hi = B.buildInstr(Opc, {SgprRB_S16}, {Op1Hi, Op2Hi}, Flags);
+ auto Lo = B.buildInstr(Opc, {{SgprRB, EltTy}}, {Op1Lo, Op2Lo}, Flags);
+ auto Hi = B.buildInstr(Opc, {{SgprRB, EltTy}}, {Op1Hi, Op2Hi}, Flags);
B.buildMergeLikeInstr(Dst, {Lo, Hi});
MI.eraseFromParent();
return true;
@@ -874,8 +876,8 @@ bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
assert(NumOps == 4);
auto [Op3Lo, Op3Hi] = unpackAExtTruncS16(MI.getOperand(3).getReg());
- auto Lo = B.buildInstr(Opc, {SgprRB_S16}, {Op1Lo, Op2Lo, Op3Lo}, Flags);
- auto Hi = B.buildInstr(Opc, {SgprRB_S16}, {Op1Hi, Op2Hi, Op3Hi}, Flags);
+ auto Lo = B.buildInstr(Opc, {{SgprRB, EltTy}}, {Op1Lo, Op2Lo, Op3Lo}, Flags);
+ auto Hi = B.buildInstr(Opc, {{SgprRB, EltTy}}, {Op1Hi, Op2Hi, Op3Hi}, Flags);
B.buildMergeLikeInstr(Dst, {Lo, Hi});
MI.eraseFromParent();
return true;
@@ -891,14 +893,14 @@ bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
// Keep the multiplication on the SALU.
- Register DstLo = B.buildMul(SgprRB_S32, Src0, Src1).getReg(0);
- Register DstHi = MRI.createVirtualRegister(SgprRB_S32);
+ Register DstLo = B.buildMul(SgprRB_I32, Src0, Src1).getReg(0);
+ Register DstHi = MRI.createVirtualRegister(SgprRB_I32);
if (ST.hasScalarMulHiInsts()) {
B.buildInstr(AMDGPU::G_UMULH, {{DstHi}}, {Src0, Src1});
} else {
- auto VSrc0 = B.buildCopy(VgprRB_S32, Src0);
- auto VSrc1 = B.buildCopy(VgprRB_S32, Src1);
- auto MulHi = B.buildInstr(AMDGPU::G_UMULH, {VgprRB_S32}, {VSrc0, VSrc1});
+ auto VSrc0 = B.buildCopy(VgprRB_I32, Src0);
+ auto VSrc1 = B.buildCopy(VgprRB_I32, Src1);
+ auto MulHi = B.buildInstr(AMDGPU::G_UMULH, {VgprRB_I32}, {VSrc0, VSrc1});
buildReadAnyLane(B, DstHi, MulHi.getReg(0), RBI);
}
@@ -913,13 +915,13 @@ bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
B.buildConstant(Dst1, 0);
} else {
// Accumulate: add Src2 to the multiplication result with carry chain.
- Register Src2Lo = MRI.createVirtualRegister(SgprRB_S32);
- Register Src2Hi = MRI.createVirtualRegister(SgprRB_S32);
+ Register Src2Lo = MRI.createVirtualRegister(SgprRB_I32);
+ Register Src2Hi = MRI.createVirtualRegister(SgprRB_I32);
B.buildUnmerge({Src2Lo, Src2Hi}, Src2);
- auto AddLo = B.buildUAddo(SgprRB_S32, SgprRB_S32, DstLo, Src2Lo);
+ auto AddLo = B.buildUAddo(SgprRB_I32, SgprRB_I32, DstLo, Src2Lo);
auto AddHi =
- B.buildUAdde(SgprRB_S32, SgprRB_S32, DstHi, Src2Hi, AddLo.getReg(1));
+ B.buildUAdde(SgprRB_I32, SgprRB_I32, DstHi, Src2Hi, AddLo.getReg(1));
B.buildMergeLikeInstr(Dst0, {AddLo.getReg(0), AddHi.getReg(0)});
B.buildCopy(Dst1, AddHi.getReg(1));
}
@@ -933,7 +935,7 @@ bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
LLT DstTy = MRI.getType(Dst);
assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
(DstTy.isPointer() && DstTy.getSizeInBits() == 64));
- LLT Ty = DstTy == V4S16 ? V2S16 : S32;
+ LLT Ty = DstTy.isFloat() ? LLT::float32() : DstTy.divide(2);
auto Op2 = B.buildUnmerge({VgprRB, Ty}, MI.getOperand(2).getReg());
auto Op3 = B.buildUnmerge({VgprRB, Ty}, MI.getOperand(3).getReg());
Register Cond = MI.getOperand(1).getReg();
@@ -949,26 +951,26 @@ bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
}
bool RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
- auto Op1 = B.buildUnmerge(VgprRB_S32, MI.getOperand(1).getReg());
+ auto Op1 = B.buildUnmerge(VgprRB_I32, MI.getOperand(1).getReg());
int Amt = MI.getOperand(2).getImm();
Register Lo, Hi;
// Hi|Lo: s sign bit, ?/x bits changed/not changed by sign-extend
if (Amt <= 32) {
- auto Freeze = B.buildFreeze(VgprRB_S32, Op1.getReg(0));
+ auto Freeze = B.buildFreeze(VgprRB_I32, Op1.getReg(0));
if (Amt == 32) {
// Hi|Lo: ????????|sxxxxxxx -> ssssssss|sxxxxxxx
Lo = Freeze.getReg(0);
} else {
// Hi|Lo: ????????|???sxxxx -> ssssssss|ssssxxxx
- Lo = B.buildSExtInReg(VgprRB_S32, Freeze, Amt).getReg(0);
+ Lo = B.buildSExtInReg(VgprRB_I32, Freeze, Amt).getReg(0);
}
- auto SignExtCst = B.buildConstant(SgprRB_S32, 31);
- Hi = B.buildAShr(VgprRB_S32, Lo, SignExtCst).getReg(0);
+ auto SignExtCst = B.buildConstant(SgprRB_I32, 31);
+ Hi = B.buildAShr(VgprRB_I32, Lo, SignExtCst).getReg(0);
} else {
// Hi|Lo: ?????sxx|xxxxxxxx -> ssssssxx|xxxxxxxx
Lo = Op1.getReg(0);
- Hi = B.buildSExtInReg(VgprRB_S32, Op1.getReg(1), Amt - 32).getReg(0);
+ Hi = B.buildSExtInReg(VgprRB_I32, Op1.getReg(1), Amt - 32).getReg(0);
}
B.buildMergeLikeInstr(MI.getOperand(0).getReg(), {Lo, Hi});
@@ -1015,19 +1017,19 @@ bool RegBankLegalizeHelper::lowerSplitBitCount64To32(MachineInstr &MI) {
llvm_unreachable("unexpected opcode in lowerSplitBitCount64To32");
}
- auto Unmerge = B.buildUnmerge(VgprRB_S32, MI.getOperand(1).getReg());
+ auto Unmerge = B.buildUnmerge(VgprRB_I32, MI.getOperand(1).getReg());
Register Lo = Unmerge.getReg(0);
Register Hi = Unmerge.getReg(1);
// MSB-first (FFBH/CTLZ) searches hi first; LSB-first (FFBL/CTTZ) searches
// lo first. The secondary half adds 32 to account for the primary half's
// width.
- auto Primary = B.buildInstr(FFBOpc, {VgprRB_S32}, {SearchFromMSB ? Hi : Lo});
+ auto Primary = B.buildInstr(FFBOpc, {VgprRB_I32}, {SearchFromMSB ? Hi : Lo});
auto Secondary =
- B.buildInstr(FFBOpc, {VgprRB_S32}, {SearchFromMSB ? Lo : Hi});
+ B.buildInstr(FFBOpc, {VgprRB_I32}, {SearchFromMSB ? Lo : Hi});
- auto Adjusted = B.buildInstr(AddOpc, {VgprRB_S32},
- {Secondary, B.buildConstant(VgprRB_S32, 32)});
+ auto Adjusted = B.buildInstr(AddOpc, {VgprRB_I32},
+ {Secondary, B.buildConstant(VgprRB_I32, 32)});
B.buildUMin(MI.getOperand(0).getReg(), Primary, Adjusted);
MI.eraseFromParent();
@@ -1064,16 +1066,16 @@ bool RegBankLegalizeHelper::lowerExtrVecEltToSel(MachineInstr &MI) {
}
B.buildCopy(Dst, PrevSelect);
} else if (ScalarTy.getSizeInBits() == 64) {
- auto InitUnmerge = B.buildUnmerge(VgprRB_S32, Unmerge.getReg(0));
+ auto InitUnmerge = B.buildUnmerge(VgprRB_I32, Unmerge.getReg(0));
Register PrevLo = InitUnmerge.getReg(0);
Register PrevHi = InitUnmerge.getReg(1);
for (unsigned I = 1; I < NumElts; ++I) {
auto IdxConst = B.buildConstant({SgprRB, MRI.getType(Idx)}, I);
auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, VccRB_S1, Idx, IdxConst);
- auto EltUnmerge = B.buildUnmerge(VgprRB_S32, Unmerge.getReg(I));
- PrevLo = B.buildSelect(VgprRB_S32, Cmp, EltUnmerge.getReg(0), PrevLo)
+ auto EltUnmerge = B.buildUnmerge(VgprRB_I32, Unmerge.getReg(I));
+ PrevLo = B.buildSelect(VgprRB_I32, Cmp, EltUnmerge.getReg(0), PrevLo)
.getReg(0);
- PrevHi = B.buildSelect(VgprRB_S32, Cmp, EltUnmerge.getReg(1), PrevHi)
+ PrevHi = B.buildSelect(VgprRB_I32, Cmp, EltUnmerge.getReg(1), PrevHi)
.getReg(0);
}
B.buildMergeLikeInstr(Dst, {PrevLo, PrevHi});
@@ -1110,12 +1112,12 @@ bool RegBankLegalizeHelper::lowerExtrVecEltTo32(MachineInstr &MI) {
auto CastSrc = B.buildBitcast({VgprRB, Vec32Ty}, Src);
// Calculate new Lo and Hi indices
- auto One = B.buildConstant(SgprRB_S32, 1);
- auto IdxLo = B.buildShl(SgprRB_S32, Idx, One);
- auto IdxHi = B.buildAdd(SgprRB_S32, IdxLo, One);
+ auto One = B.buildConstant(SgprRB_I32, 1);
+ auto IdxLo = B.buildShl(SgprRB_I32, Idx, One);
+ auto IdxHi = B.buildAdd(SgprRB_I32, IdxLo, One);
- auto ExtLo = B.buildExtractVectorElement(VgprRB_S32, CastSrc, IdxLo);
- auto ExtHi = B.buildExtractVectorElement(VgprRB_S32, CastSrc, IdxHi);
+ auto ExtLo = B.buildExtractVectorElement(VgprRB_I32, CastSrc, IdxLo);
+ auto ExtHi = B.buildExtractVectorElement(VgprRB_I32, CastSrc, IdxHi);
B.buildMergeLikeInstr(Dst, {ExtLo.getReg(0), ExtHi.getReg(0)});
@@ -1145,18 +1147,18 @@ bool RegBankLegalizeHelper::lowerInsVecEltToSel(MachineInstr &MI) {
if (!IsSGPR && ScalarTy.getSizeInBits() == 64) {
// VGPR B64: split to 32-bit lo/hi since there is no v_cndmask_b64.
- auto Unmerge = B.buildUnmerge(VgprRB_S32, Src);
- auto EltUnmerge = B.buildUnmerge(VgprRB_S32, Elt);
+ auto Unmerge = B.buildUnmerge(VgprRB_I32, Src);
+ auto EltUnmerge = B.buildUnmerge(VgprRB_I32, Elt);
Register EltLo = EltUnmerge.getReg(0);
Register EltHi = EltUnmerge.getReg(1);
for (unsigned I = 0; I < NumElts; ++I) {
- auto IdxConst = B.buildConstant(VgprRB_S32, I);
+ auto IdxConst = B.buildConstant(VgprRB_I32, I);
auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, VccRB_S1, Idx, IdxConst);
Selects.push_back(
- B.buildSelect(VgprRB_S32, Cmp, EltLo, Unmerge.getReg(2 * I))
+ B.buildSelect(VgprRB_I32, Cmp, EltLo, Unmerge.getReg(2 * I))
.getReg(0));
Selects.push_back(
- B.buildSelect(VgprRB_S32, Cmp, EltHi, Unmerge.getReg(2 * I + 1))
+ B.buildSelect(VgprRB_I32, Cmp, EltHi, Unmerge.getReg(2 * I + 1))
.getReg(0));
}
LLT Vec32Ty = LLT::fixed_vector(2 * NumElts, 32);
@@ -1165,10 +1167,10 @@ bool RegBankLegalizeHelper::lowerInsVecEltToSel(MachineInstr &MI) {
} else if (ScalarTy.getSizeInBits() == 32 || ScalarTy.getSizeInBits() == 64) {
// B32 (any bank) and SGPR B64: element-wise select at native width.
MachineRegisterInfo::VRegAttrs SrcRB_EltTy = {SrcRB, ScalarTy};
- MachineRegisterInfo::VRegAttrs CmpTy = IsSGPR ? SgprRB_S32 : VccRB_S1;
+ MachineRegisterInfo::VRegAttrs CmpTy = IsSGPR ? SgprRB_I32 : VccRB_S1;
auto Unmerge = B.buildUnmerge(SrcRB_EltTy, Src);
for (unsigned I = 0; I < NumElts; ++I) {
- auto IdxConst = B.buildConstant(SgprRB_S32, I);
+ auto IdxConst = B.buildConstant(SgprRB_I32, I);
auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CmpTy, Idx, IdxConst);
Selects.push_back(
B.buildSelect(SrcRB_EltTy, Cmp, Elt, Unmerge.getReg(I)).getReg(0));
@@ -1209,12 +1211,12 @@ bool RegBankLegalizeHelper::lowerInsVecEltTo32(MachineInstr &MI) {
MachineRegisterInfo::VRegAttrs VgprRB_Vec32Ty = {VgprRB, Vec32Ty};
auto CastSrc = B.buildBitcast(VgprRB_Vec32Ty, Src);
- auto EltUnmerge = B.buildUnmerge(VgprRB_S32, Elt);
+ auto EltUnmerge = B.buildUnmerge(VgprRB_I32, Elt);
// Calculate new Lo and Hi indices
- auto One = B.buildConstant(SgprRB_S32, 1);
- auto IdxLo = B.buildShl(SgprRB_S32, Idx, One);
- auto IdxHi = B.buildAdd(SgprRB_S32, IdxLo, One);
+ auto One = B.buildConstant(SgprRB_I32, 1);
+ auto IdxLo = B.buildShl(SgprRB_I32, Idx, One);
+ auto IdxHi = B.buildAdd(SgprRB_I32, IdxLo, One);
auto InsLo = B.buildInsertVectorElement(VgprRB_Vec32Ty, CastSrc,
EltUnmerge.getReg(0), IdxLo);
@@ -1242,7 +1244,7 @@ bool RegBankLegalizeHelper::lowerAbsToNegMax(MachineInstr &MI) {
Register Zero;
if (Ty == V2S16) {
// buildConstant cannot produce a V2S16 directly; pack two S16 zeros.
- Register Zero16 = B.buildConstant({VgprRB, S16}, 0).getReg(0);
+ Register Zero16 = B.buildConstant({VgprRB, I16}, 0).getReg(0);
Zero = B.buildBuildVector({VgprRB, Ty}, {Zero16, Zero16}).getReg(0);
} else {
assert((Ty == S32 || Ty == S16) && "unexpected type for AbsToNegMax");
@@ -1265,13 +1267,13 @@ bool RegBankLegalizeHelper::lowerAbsToS32(MachineInstr &MI) {
//
// SALU only has s_abs_i32, with no direct uniform V2S16 abs. The
// re-emitted G_ABS(SgprRB, S32) selects to s_abs_i32 on each value.
- auto Bitcast = B.buildBitcast({SgprRB_S32}, MI.getOperand(1).getReg());
- auto SextInReg = B.buildSExtInReg({SgprRB_S32}, Bitcast, 16);
+ auto Bitcast = B.buildBitcast({SgprRB_I32}, MI.getOperand(1).getReg());
+ auto SextInReg = B.buildSExtInReg({SgprRB_I32}, Bitcast, 16);
auto ShiftHi =
- B.buildAShr({SgprRB_S32}, Bitcast, B.buildConstant({SgprRB_S32}, 16));
+ B.buildAShr({SgprRB_I32}, Bitcast, B.buildConstant({SgprRB_I32}, 16));
- auto AbsLo = B.buildInstr(AMDGPU::G_ABS, {{SgprRB_S32}}, {SextInReg});
- auto AbsHi = B.buildInstr(AMDGPU::G_ABS, {{SgprRB_S32}}, {ShiftHi});
+ auto AbsLo = B.buildInstr(AMDGPU::G_ABS, {{SgprRB_I32}}, {SextInReg});
+ auto AbsHi = B.buildInstr(AMDGPU::G_ABS, {{SgprRB_I32}}, {ShiftHi});
B.buildBuildVectorTrunc(MI.getOperand(0).getReg(),
{AbsLo.getReg(0), AbsHi.getReg(0)});
@@ -1291,7 +1293,7 @@ bool RegBankLegalizeHelper::lowerSetRounding(MachineInstr &MI) {
static_cast<uint32_t>(ConstMode->Value.getZExtValue()),
static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
uint32_t DecodedVal = AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal);
- NewMode = B.buildConstant(SgprRB_S32, DecodedVal).getReg(0);
+ NewMode = B.buildConstant(SgprRB_I32, DecodedVal).getReg(0);
} else {
// If we know the input can only be one of the supported standard modes in
// the range 0-3, we can use a simplified mapping to hardware values.
@@ -1303,33 +1305,33 @@ bool RegBankLegalizeHelper::lowerSetRounding(MachineInstr &MI) {
if (UseReducedTable) {
// Truncate to the low 32-bits.
auto BitTable = B.buildConstant(
- SgprRB_S32, AMDGPU::FltRoundToHWConversionTable & 0xffff);
+ SgprRB_I32, AMDGPU::FltRoundToHWConversionTable & 0xffff);
- auto Two = B.buildConstant(SgprRB_S32, 2);
- auto RoundModeTimesNumBits = B.buildShl(SgprRB_S32, NewMode, Two);
+ auto Two = B.buildConstant(SgprRB_I32, 2);
+ auto RoundModeTimesNumBits = B.buildShl(SgprRB_I32, NewMode, Two);
NewMode =
- B.buildLShr(SgprRB_S32, BitTable, RoundModeTimesNumBits).getReg(0);
+ B.buildLShr(SgprRB_I32, BitTable, RoundModeTimesNumBits).getReg(0);
// TODO: A demanded-bits simplification on the setreg source here could
// likely reduce the table extracted bits into inline immediates.
} else {
// table_index = umin(value, value - 4)
// MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
- auto NegFour = B.buildConstant(SgprRB_S32, -4);
- auto OffsetEnum = B.buildAdd(SgprRB_S32, NewMode, NegFour);
- auto IndexVal = B.buildUMin(SgprRB_S32, NewMode, OffsetEnum);
+ auto NegFour = B.buildConstant(SgprRB_I32, -4);
+ auto OffsetEnum = B.buildAdd(SgprRB_I32, NewMode, NegFour);
+ auto IndexVal = B.buildUMin(SgprRB_I32, NewMode, OffsetEnum);
- auto Two = B.buildConstant(SgprRB_S32, 2);
- auto RoundModeTimesNumBits = B.buildShl(SgprRB_S32, IndexVal, Two);
+ auto Two = B.buildConstant(SgprRB_I32, 2);
+ auto RoundModeTimesNumBits = B.buildShl(SgprRB_I32, IndexVal, Two);
auto BitTable =
- B.buildConstant({SgprRB, S64}, AMDGPU::FltRoundToHWConversionTable);
+ B.buildConstant(SgprRB_I64, AMDGPU::FltRoundToHWConversionTable);
auto TableValue =
- B.buildLShr({SgprRB, S64}, BitTable, RoundModeTimesNumBits);
+ B.buildLShr(SgprRB_I64, BitTable, RoundModeTimesNumBits);
// No need to mask out the high bits since the setreg will ignore them
// anyway.
- NewMode = B.buildTrunc(SgprRB_S32, TableValue).getReg(0);
+ NewMode = B.buildTrunc(SgprRB_I32, TableValue).getReg(0);
}
}
@@ -1354,7 +1356,7 @@ bool RegBankLegalizeHelper::lowerGetRounding(MachineInstr &MI) {
uint32_t BothRoundHwReg =
AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 4);
auto GetReg =
- B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {SgprRB_S32},
+ B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {SgprRB_I32},
/*HasSideEffects=*/true, /*isConvergent=*/false)
.addImm(BothRoundHwReg);
@@ -1385,26 +1387,25 @@ bool RegBankLegalizeHelper::lowerGetRounding(MachineInstr &MI) {
// table we can index by the raw hardware mode.
//
// (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
- auto BitTable =
- B.buildConstant({SgprRB, S64}, AMDGPU::FltRoundConversionTable);
+ auto BitTable = B.buildConstant(SgprRB_I64, AMDGPU::FltRoundConversionTable);
- auto Two = B.buildConstant(SgprRB_S32, 2);
- auto RoundModeTimesNumBits = B.buildShl(SgprRB_S32, GetReg, Two);
+ auto Two = B.buildConstant(SgprRB_I32, 2);
+ auto RoundModeTimesNumBits = B.buildShl(SgprRB_I32, GetReg, Two);
// TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
// knew only one mode was demanded.
- auto TableValue = B.buildLShr({SgprRB, S64}, BitTable, RoundModeTimesNumBits);
- auto TruncTable = B.buildTrunc(SgprRB_S32, TableValue);
+ auto TableValue = B.buildLShr(SgprRB_I64, BitTable, RoundModeTimesNumBits);
+ auto TruncTable = B.buildTrunc(SgprRB_I32, TableValue);
- auto EntryMask = B.buildConstant(SgprRB_S32, 0xf);
- auto TableEntry = B.buildAnd(SgprRB_S32, TruncTable, EntryMask);
+ auto EntryMask = B.buildConstant(SgprRB_I32, 0xf);
+ auto TableEntry = B.buildAnd(SgprRB_I32, TruncTable, EntryMask);
// There's a gap in the 4-bit encoded table and actual enum values, so offset
// if it's an extended value.
- auto Four = B.buildConstant(SgprRB_S32, 4);
- auto EnumOffset = B.buildAdd(SgprRB_S32, TableEntry, Four);
+ auto Four = B.buildConstant(SgprRB_I32, 4);
+ auto EnumOffset = B.buildAdd(SgprRB_I32, TableEntry, Four);
auto IsStandardMode =
- B.buildICmp(CmpInst::ICMP_ULT, SgprRB_S32, TableEntry, Four);
+ B.buildICmp(CmpInst::ICMP_ULT, SgprRB_I32, TableEntry, Four);
B.buildSelect(Dst, IsStandardMode, TableEntry, EnumOffset);
MI.eraseFromParent();
@@ -1444,17 +1445,18 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
MachineInstrBuilder Hi;
switch (MI.getOpcode()) {
case AMDGPU::G_ZEXT: {
- Hi = B.buildConstant({RB, S32}, 0);
+ Hi = B.buildConstant({RB, I32}, 0);
break;
}
case AMDGPU::G_SEXT: {
// Replicate sign bit from 32-bit extended part.
- auto ShiftAmt = B.buildConstant({RB, S32}, 31);
- Hi = B.buildAShr({RB, S32}, MI.getOperand(1).getReg(), ShiftAmt);
+ auto ShiftAmt = B.buildConstant({RB, I32}, 31);
+ Hi = B.buildAShr({RB, MRI.getType(MI.getOperand(1).getReg())},
+ MI.getOperand(1).getReg(), ShiftAmt);
break;
}
case AMDGPU::G_ANYEXT: {
- Hi = B.buildUndef({RB, S32});
+ Hi = B.buildUndef({RB, I32});
break;
}
default:
@@ -1484,11 +1486,11 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
// compares all bits in register.
Register BoolSrc = MRI.createVirtualRegister({VgprRB, Ty});
if (Ty == S64) {
- auto Src64 = B.buildUnmerge(VgprRB_S32, Src);
- auto One = B.buildConstant(VgprRB_S32, 1);
- auto AndLo = B.buildAnd(VgprRB_S32, Src64.getReg(0), One);
- auto Zero = B.buildConstant(VgprRB_S32, 0);
- auto AndHi = B.buildAnd(VgprRB_S32, Src64.getReg(1), Zero);
+ auto Src64 = B.buildUnmerge(VgprRB_I32, Src);
+ auto One = B.buildConstant(VgprRB_I32, 1);
+ auto AndLo = B.buildAnd(VgprRB_I32, Src64.getReg(0), One);
+ auto Zero = B.buildConstant(VgprRB_I32, 0);
+ auto AndHi = B.buildAnd(VgprRB_I32, Src64.getReg(1), Zero);
B.buildMergeLikeInstr(BoolSrc, {AndLo, AndHi});
} else {
assert(Ty == S32 || Ty == S16);
@@ -1512,15 +1514,15 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return true;
}
case DivSMulToMAD: {
- auto Op1 = B.buildTrunc(VgprRB_S32, MI.getOperand(1));
- auto Op2 = B.buildTrunc(VgprRB_S32, MI.getOperand(2));
- auto Zero = B.buildConstant({VgprRB, S64}, 0);
+ auto Op1 = B.buildTrunc(VgprRB_I32, MI.getOperand(1));
+ auto Op2 = B.buildTrunc(VgprRB_I32, MI.getOperand(2));
+ auto Zero = B.buildConstant(VgprRB_I64, 0);
unsigned NewOpc = MI.getOpcode() == AMDGPU::G_AMDGPU_S_MUL_U64_U32
? AMDGPU::G_AMDGPU_MAD_U64_U32
: AMDGPU::G_AMDGPU_MAD_I64_I32;
- B.buildInstr(NewOpc, {MI.getOperand(0).getReg(), {SgprRB, S32}},
+ B.buildInstr(NewOpc, {MI.getOperand(0).getReg(), SgprRB_I32},
{Op1, Op2, Zero});
MI.eraseFromParent();
return true;
@@ -1534,9 +1536,9 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
case SplitTo32SExtInReg:
return lowerSplitTo32SExtInReg(MI);
case CtPop64To32: {
- auto Unmerge = B.buildUnmerge({VgprRB, S32}, MI.getOperand(1).getReg());
- auto LoPopCnt = B.buildCTPOP({VgprRB, S32}, Unmerge.getReg(0));
- auto HiPopCnt = B.buildCTPOP({VgprRB, S32}, Unmerge.getReg(1));
+ auto Unmerge = B.buildUnmerge(VgprRB_I32, MI.getOperand(1).getReg());
+ auto LoPopCnt = B.buildCTPOP(VgprRB_I32, Unmerge.getReg(0));
+ auto HiPopCnt = B.buildCTPOP(VgprRB_I32, Unmerge.getReg(1));
// Max popcount of two 32-bit values is 64, so this add cannot overflow.
B.buildAdd(MI.getOperand(0).getReg(), LoPopCnt, HiPopCnt,
MachineInstr::NoSWrap | MachineInstr::NoUWrap);
@@ -1556,7 +1558,7 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
LLT EltTy = DstTy.getElementType();
B128 = LLT::fixed_vector(128 / EltTy.getSizeInBits(), EltTy);
} else {
- B128 = LLT::scalar(128);
+ B128 = LLT::integer(128);
}
if (Size / 128 == 2)
splitLoad(MI, {B128, B128});
@@ -1602,7 +1604,7 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
if (MRI.getRegBank(AllocSize) != SgprRB) {
auto WaveReduction =
- B.buildIntrinsic(Intrinsic::amdgcn_wave_reduce_umax, {SgprRB_S32})
+ B.buildIntrinsic(Intrinsic::amdgcn_wave_reduce_umax, {SgprRB_I32})
.addUse(AllocSize)
.addImm(0);
AllocSize = WaveReduction.getReg(0);
@@ -1620,8 +1622,8 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
Register AdjustedSize = AllocSize;
if (!HasFlatScratch) {
- auto WaveSize = B.buildConstant(SgprRB_S32, WavefrontSizeLog2);
- AdjustedSize = B.buildShl(SgprRB_S32, AllocSize, WaveSize).getReg(0);
+ auto WaveSize = B.buildConstant(SgprRB_I32, WavefrontSizeLog2);
+ AdjustedSize = B.buildShl(SgprRB_I32, AllocSize, WaveSize).getReg(0);
}
if (Alignment > TFI.getStackAlign()) {
const uint64_t EffectiveAlignment =
@@ -1629,9 +1631,9 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
auto OldSP = B.buildCopy({SgprRB, PtrTy}, SPReg);
auto Tmp1 =
B.buildPtrAdd({SgprRB, PtrTy}, OldSP,
- B.buildConstant(SgprRB_S32, EffectiveAlignment - 1));
+ B.buildConstant(SgprRB_I32, EffectiveAlignment - 1));
uint64_t Mask = maskTrailingZeros<uint64_t>(Log2_64(EffectiveAlignment));
- B.buildPtrMask(Dst, Tmp1, B.buildConstant(SgprRB_S32, Mask));
+ B.buildPtrMask(Dst, Tmp1, B.buildConstant(SgprRB_I32, Mask));
} else {
B.buildCopy(Dst, SPReg);
}
@@ -1697,15 +1699,15 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
auto UnmergeV2S16 =
B.buildUnmerge({SgprRB, V2S16}, Unmerge->getSourceReg());
for (unsigned i = 0; i < UnmergeV2S16->getNumDefs(); ++i) {
- auto [Dst0S32, Dst1S32] =
+ auto [Dst0I32, Dst1I32] =
unpackAExt(UnmergeV2S16->getOperand(i).getReg());
- B.buildTrunc(MI.getOperand(i * 2).getReg(), Dst0S32);
- B.buildTrunc(MI.getOperand(i * 2 + 1).getReg(), Dst1S32);
+ B.buildTrunc(MI.getOperand(i * 2).getReg(), Dst0I32);
+ B.buildTrunc(MI.getOperand(i * 2 + 1).getReg(), Dst1I32);
}
} else {
- auto [Dst0S32, Dst1S32] = unpackAExt(MI.getOperand(2).getReg());
- B.buildTrunc(MI.getOperand(0).getReg(), Dst0S32);
- B.buildTrunc(MI.getOperand(1).getReg(), Dst1S32);
+ auto [Dst0I32, Dst1I32] = unpackAExt(MI.getOperand(2).getReg());
+ B.buildTrunc(MI.getOperand(0).getReg(), Dst0I32);
+ B.buildTrunc(MI.getOperand(1).getReg(), Dst1I32);
}
MI.eraseFromParent();
@@ -1713,7 +1715,7 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
}
case AextToS32InIncomingBlockGPHI: {
Register Dst = MI.getOperand(0).getReg();
- Register NewDst = MRI.createVirtualRegister(SgprRB_S32);
+ Register NewDst = MRI.createVirtualRegister(SgprRB_I32);
B.setInsertPt(*MI.getParent(), MI.getParent()->getFirstNonPHI());
MI.getOperand(0).setReg(NewDst);
B.buildTrunc(Dst, NewDst);
@@ -1726,7 +1728,7 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
B.setInsertPt(*DefMBB, DefMBB->SkipPHIsAndLabels(std::next(DefMI)));
- auto NewUse = B.buildAnyExt(SgprRB_S32, UseReg);
+ auto NewUse = B.buildAnyExt(SgprRB_I32, UseReg);
MI.getOperand(i).setReg(NewUse.getReg(0));
}
break;
@@ -2206,7 +2208,7 @@ bool RegBankLegalizeHelper::applyMappingDst(
Op.setReg(NewDst);
if (!MRI.use_empty(Reg)) {
auto CopyS32_Vcc =
- B.buildInstr(AMDGPU::G_AMDGPU_COPY_SCC_VCC, {SgprRB_S32}, {NewDst});
+ B.buildInstr(AMDGPU::G_AMDGPU_COPY_SCC_VCC, {SgprRB_I32}, {NewDst});
B.buildTrunc(Reg, CopyS32_Vcc);
}
break;
@@ -2214,13 +2216,13 @@ bool RegBankLegalizeHelper::applyMappingDst(
case UniInVgprS16: {
assert(Ty == getTyFromID(MethodIDs[OpIdx]));
assert(RB == SgprRB);
- Register NewVgprDstS16 = MRI.createVirtualRegister({VgprRB, S16});
- Register NewVgprDstS32 = MRI.createVirtualRegister({VgprRB, S32});
- Register NewSgprDstS32 = MRI.createVirtualRegister({SgprRB, S32});
- Op.setReg(NewVgprDstS16);
- B.buildAnyExt(NewVgprDstS32, NewVgprDstS16);
- buildReadAnyLane(B, NewSgprDstS32, NewVgprDstS32, RBI);
- B.buildTrunc(Reg, NewSgprDstS32);
+ Register NewVgprDst16 = MRI.createVirtualRegister({VgprRB, Ty});
+ Register NewVgprDstI32 = MRI.createVirtualRegister(VgprRB_I32);
+ Register NewSgprDstI32 = MRI.createVirtualRegister(SgprRB_I32);
+ Op.setReg(NewVgprDst16);
+ B.buildAnyExt(NewVgprDstI32, NewVgprDst16);
+ buildReadAnyLane(B, NewSgprDstI32, NewVgprDstI32, RBI);
+ B.buildTrunc(Reg, NewSgprDstI32);
break;
}
case UniInVgprS32:
@@ -2262,7 +2264,7 @@ bool RegBankLegalizeHelper::applyMappingDst(
case Sgpr32Trunc: {
assert(Ty.getSizeInBits() < 32);
assert(RB == SgprRB);
- Register NewDst = MRI.createVirtualRegister(SgprRB_S32);
+ Register NewDst = MRI.createVirtualRegister(SgprRB_I32);
Op.setReg(NewDst);
if (!MRI.use_empty(Reg))
B.buildTrunc(Reg, NewDst);
@@ -2311,7 +2313,7 @@ bool RegBankLegalizeHelper::applyMappingSrc(
assert(Ty == S1);
assert(RB == VccRB || RB == SgprRB);
if (RB == SgprRB) {
- auto Aext = B.buildAnyExt(SgprRB_S32, Reg);
+ auto Aext = B.buildAnyExt(SgprRB_I32, Reg);
auto CopyVcc_Scc =
B.buildInstr(AMDGPU::G_AMDGPU_COPY_VCC_SCC, {VccRB_S1}, {Aext});
Op.setReg(CopyVcc_Scc.getReg(0));
@@ -2487,7 +2489,7 @@ bool RegBankLegalizeHelper::applyMappingSrc(
// Note: this ext allows S1, and it is meant to be combined away.
assert(Ty.getSizeInBits() < 32);
assert(RB == SgprRB);
- auto Aext = B.buildAnyExt(SgprRB_S32, Reg);
+ auto Aext = B.buildAnyExt(SgprRB_I32, Reg);
Op.setReg(Aext.getReg(0));
break;
}
@@ -2495,32 +2497,32 @@ bool RegBankLegalizeHelper::applyMappingSrc(
// Note: this ext allows S1, and it is meant to be combined away.
assert(Ty.getSizeInBits() == 1);
assert(RB == SgprRB);
- auto Aext = B.buildAnyExt(SgprRB_S32, Reg);
+ auto Aext = B.buildAnyExt(SgprRB_I32, Reg);
// Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
// most of times meant to be combined away in AMDGPURegBankCombiner.
- auto Cst1 = B.buildConstant(SgprRB_S32, 1);
- auto BoolInReg = B.buildAnd(SgprRB_S32, Aext, Cst1);
+ auto Cst1 = B.buildConstant(SgprRB_I32, 1);
+ auto BoolInReg = B.buildAnd(SgprRB_I32, Aext, Cst1);
Op.setReg(BoolInReg.getReg(0));
break;
}
case Sgpr32SExt: {
assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
assert(RB == SgprRB);
- auto Sext = B.buildSExt(SgprRB_S32, Reg);
+ auto Sext = B.buildSExt(SgprRB_I32, Reg);
Op.setReg(Sext.getReg(0));
break;
}
case Sgpr32ZExt: {
assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
assert(RB == SgprRB);
- auto Zext = B.buildZExt({SgprRB, S32}, Reg);
+ auto Zext = B.buildZExt(SgprRB_I32, Reg);
Op.setReg(Zext.getReg(0));
break;
}
case Vgpr32AExt: {
assert(Ty.getSizeInBits() < 32);
assert(RB == VgprRB);
- auto Aext = B.buildAnyExt({VgprRB, S32}, Reg);
+ auto Aext = B.buildAnyExt(VgprRB_I32, Reg);
Op.setReg(Aext.getReg(0));
break;
}
@@ -2528,7 +2530,7 @@ bool RegBankLegalizeHelper::applyMappingSrc(
// Note this ext allows S1, and it is meant to be combined away.
assert(Ty.getSizeInBits() < 32);
assert(RB == VgprRB);
- auto Sext = B.buildSExt({VgprRB, S32}, Reg);
+ auto Sext = B.buildSExt(VgprRB_I32, Reg);
Op.setReg(Sext.getReg(0));
break;
}
@@ -2536,7 +2538,7 @@ bool RegBankLegalizeHelper::applyMappingSrc(
// Note this ext allows S1, and it is meant to be combined away.
assert(Ty.getSizeInBits() < 32);
assert(RB == VgprRB);
- auto Zext = B.buildZExt({VgprRB, S32}, Reg);
+ auto Zext = B.buildZExt(VgprRB_I32, Reg);
Op.setReg(Zext.getReg(0));
break;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
index 25c8c3a0f6127..11a09f7f87cba 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
@@ -90,9 +90,14 @@ class RegBankLegalizeHelper {
static constexpr LLT P4 = LLT::pointer(4, 64);
static constexpr LLT P6 = LLT::pointer(6, 32);
- MachineRegisterInfo::VRegAttrs SgprRB_S32 = {SgprRB, S32};
- MachineRegisterInfo::VRegAttrs SgprRB_S16 = {SgprRB, S16};
- MachineRegisterInfo::VRegAttrs VgprRB_S32 = {VgprRB, S32};
+ const LLT I16 = LLT::integer(16);
+ const LLT I32 = LLT::integer(32);
+ const LLT I64 = LLT::integer(64);
+
+ MachineRegisterInfo::VRegAttrs SgprRB_I32 = {SgprRB, I32};
+ MachineRegisterInfo::VRegAttrs SgprRB_I64 = {SgprRB, I64};
+ MachineRegisterInfo::VRegAttrs VgprRB_I32 = {VgprRB, I32};
+ MachineRegisterInfo::VRegAttrs VgprRB_I64 = {VgprRB, I64};
MachineRegisterInfo::VRegAttrs VccRB_S1 = {VccRB, S1};
public:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll
index 76a2716d8d4d9..a239222f8b5e4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
define i16 @s_add_i16(i16 inreg %a, i16 inreg %b) {
; GFX7-LABEL: s_add_i16:
@@ -213,8 +213,12 @@ define <2 x i16> @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
; GFX9-LABEL: s_add_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_pk_add_u16 v0, s16, v0
+; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: s_lshr_b32 s5, s17, 16
+; GFX9-NEXT: s_add_i32 s16, s16, s17
+; GFX9-NEXT: s_add_i32 s4, s4, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s4
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: s_add_v2i16:
@@ -233,13 +237,23 @@ define <2 x i16> @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
; GFX10-LABEL: s_add_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_add_u16 v0, s16, s17
+; GFX10-NEXT: s_lshr_b32 s4, s16, 16
+; GFX10-NEXT: s_lshr_b32 s5, s17, 16
+; GFX10-NEXT: s_add_i32 s16, s16, s17
+; GFX10-NEXT: s_add_i32 s4, s4, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s4, s16, s4
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: s_add_v2i16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_add_u16 v0, s0, s1
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: s_add_v2i16:
@@ -249,7 +263,15 @@ define <2 x i16> @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_pk_add_u16 v0, s0, s1
+; GFX12-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-NEXT: s_lshr_b32 s3, s1, 16
+; GFX12-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s2, s2, s3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%c = add <2 x i16> %a, %b
ret <2 x i16> %c
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
index a22c0b5932d7f..94fd7634201ff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX10 %s
define <2 x i16> @v_add_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX7-LABEL: v_add_v2i16:
@@ -281,8 +281,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
;
; GFX9-LABEL: s_add_v2i16_neg_inline_imm_splat:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_sub_u16 v0, s0, 64 op_sel_hi:[1,0]
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_add_i32 s0, s0, 0xffc0ffc0
+; GFX9-NEXT: s_add_i32 s1, s1, 0xffc0
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_add_v2i16_neg_inline_imm_splat:
@@ -296,8 +298,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
;
; GFX10-LABEL: s_add_v2i16_neg_inline_imm_splat:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_sub_u16 v0, s0, 64 op_sel_hi:[1,0]
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-NEXT: s_add_i32 s0, s0, 0xffc0ffc0
+; GFX10-NEXT: s_add_i32 s1, s1, 0xffc0
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
%add = add <2 x i16> %a, <i16 -64, i16 -64>
%cast = bitcast <2 x i16> %add to i32
@@ -316,9 +320,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
;
; GFX9-LABEL: s_add_v2i16_neg_inline_imm_lo:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x4ffc0
-; GFX9-NEXT: v_pk_add_u16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_add_i32 s0, s0, 0x4ffc0
+; GFX9-NEXT: s_add_i32 s1, s1, 4
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_add_v2i16_neg_inline_imm_lo:
@@ -332,8 +337,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
;
; GFX10-LABEL: s_add_v2i16_neg_inline_imm_lo:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_add_u16 v0, 0x4ffc0, s0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-NEXT: s_add_i32 s0, s0, 0x4ffc0
+; GFX10-NEXT: s_add_i32 s1, s1, 4
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
%add = add <2 x i16> %a, <i16 -64, i16 4>
%cast = bitcast <2 x i16> %add to i32
@@ -352,9 +359,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
;
; GFX9-LABEL: s_add_v2i16_neg_inline_imm_hi:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, 0xffc00004
-; GFX9-NEXT: v_pk_add_u16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_add_i32 s0, s0, 0xffc00004
+; GFX9-NEXT: s_add_i32 s1, s1, 0xffc0
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_add_v2i16_neg_inline_imm_hi:
@@ -368,8 +376,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
;
; GFX10-LABEL: s_add_v2i16_neg_inline_imm_hi:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_add_u16 v0, 0xffc00004, s0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-NEXT: s_add_i32 s0, s0, 0xffc00004
+; GFX10-NEXT: s_add_i32 s1, s1, 0xffc0
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
%add = add <2 x i16> %a, <i16 4, i16 -64>
%cast = bitcast <2 x i16> %add to i32
@@ -390,9 +400,11 @@ define amdgpu_ps i32 @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
;
; GFX9-LABEL: s_add_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_add_u16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_add_i32 s2, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_add_v2i16:
@@ -408,8 +420,11 @@ define amdgpu_ps i32 @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
;
; GFX10-LABEL: s_add_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_add_u16 v0, s0, s1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_add_i32 s0, s0, s1
+; GFX10-NEXT: s_add_i32 s2, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
%add = add <2 x i16> %a, %b
%cast = bitcast <2 x i16> %add to i32
@@ -431,10 +446,14 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg
;
; GFX9-LABEL: s_add_v2i16_fneg_lhs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_add_u16 v0, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_add_i32 s2, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_add_v2i16_fneg_lhs:
@@ -451,9 +470,13 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg
;
; GFX10-LABEL: s_add_v2i16_fneg_lhs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX10-NEXT: v_pk_add_u16 v0, s0, s1
+; GFX10-NEXT: v_xor_b32_e64 v0, 0x80008000, s0
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_add_i32 s0, s0, s1
+; GFX10-NEXT: s_add_i32 s2, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
%neg.a = fneg <2 x half> %a
%cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -477,10 +500,14 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg
;
; GFX9-LABEL: s_add_v2i16_fneg_rhs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s1, 0x80008000
; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_add_u16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_add_i32 s2, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_add_v2i16_fneg_rhs:
@@ -497,9 +524,13 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg
;
; GFX10-LABEL: s_add_v2i16_fneg_rhs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX10-NEXT: v_pk_add_u16 v0, s0, s1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_xor_b32_e64 v0, 0x80008000, s1
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: v_readfirstlane_b32 s1, v0
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_add_i32 s0, s0, s1
+; GFX10-NEXT: s_add_i32 s2, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
%neg.b = fneg <2 x half> %b
%cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
@@ -524,11 +555,17 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> inreg %a, <2 x ha
;
; GFX9-LABEL: s_add_v2i16_fneg_lhs_fneg_rhs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX9-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_add_u16 v0, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_add_i32 s2, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_add_v2i16_fneg_lhs_fneg_rhs:
@@ -546,10 +583,15 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> inreg %a, <2 x ha
;
; GFX10-LABEL: s_add_v2i16_fneg_lhs_fneg_rhs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX10-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX10-NEXT: v_pk_add_u16 v0, s0, s1
+; GFX10-NEXT: v_xor_b32_e64 v0, 0x80008000, s0
+; GFX10-NEXT: v_xor_b32_e64 v1, 0x80008000, s1
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_add_i32 s0, s0, s1
+; GFX10-NEXT: s_add_i32 s2, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
%neg.a = fneg <2 x half> %a
%neg.b = fneg <2 x half> %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
index 6fc30daf64cac..03ab046a1d2b2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
define amdgpu_ps i16 @s_and_i16(i16 inreg %num, i16 inreg %den) {
; GCN-LABEL: s_and_i16:
@@ -532,8 +532,7 @@ define amdgpu_ps void @s_and_u64_zext_with_vregs(ptr addrspace(1) %out, ptr addr
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: buffer_load_dword v2, v[2:3], s[0:3], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v3, 0
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -687,55 +686,60 @@ define amdgpu_ps void @s_and_u64_sext_with_vregs(ptr addrspace(1) %out, ptr addr
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
-; GFX7-NEXT: buffer_load_dword v2, v[2:3], s[0:3], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
+; GFX7-NEXT: v_mov_b32_e32 v2, 0x50
; GFX7-NEXT: v_mov_b32_e32 v3, 0
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v2, 0x50, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0x50, v4
; GFX7-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_and_u64_sext_with_vregs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v2, v[2:3]
+; GFX8-NEXT: flat_load_dword v4, v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, 0x50
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v2, 0x50, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0x50, v4
; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: s_and_u64_sext_with_vregs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: global_load_dword v2, v[2:3], off
+; GFX9-NEXT: global_load_dword v4, v[2:3], off
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x50
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, 0x50, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 0x50, v4
; GFX9-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_and_u64_sext_with_vregs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: global_load_dword v2, v[2:3], off
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: v_mov_b32_e32 v2, 0x50
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v2, 0x50, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0x50, v4
; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_and_u64_sext_with_vregs:
; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v2, v[2:3], off
+; GFX11-NEXT: global_load_b32 v4, v[2:3], off
+; GFX11-NEXT: v_dual_mov_b32 v2, 0x50 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v2, 0x50, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0x50, v4
; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_and_u64_sext_with_vregs:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-NEXT: global_load_b32 v4, v[2:3], off
+; GFX12-NEXT: v_dual_mov_b32 v2, 0x50 :: v_dual_mov_b32 v3, 0
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v2, 0x50, v2
+; GFX12-NEXT: v_and_b32_e32 v2, 0x50, v4
; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
@@ -749,76 +753,86 @@ define amdgpu_kernel void @s_and_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX7-LABEL: s_and_u64_sext_with_sregs:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s4, s4, 0x50
+; GFX7-NEXT: s_ashr_i32 s5, s4, 31
+; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], 0x50
; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_and_u64_sext_with_sregs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_load_dword s2, s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s0, s2, 0x50
; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_ashr_i32 s3, s2, 31
+; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], 0x50
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: s_and_u64_sext_with_sregs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s2, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s2, s2, 0x50
+; GFX9-NEXT: s_ashr_i32 s3, s2, 31
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], 0x50
; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_and_u64_sext_with_sregs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s2, s[2:3], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_and_b32 s2, s2, 0x50
+; GFX10-NEXT: s_ashr_i32 s3, s2, 31
+; GFX10-NEXT: s_and_b64 s[2:3], s[2:3], 0x50
; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_and_u64_sext_with_sregs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s2, s2, 0x50
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX11-NEXT: global_store_b64 v1, v[0:1], s[0:1]
+; GFX11-NEXT: s_ashr_i32 s3, s2, 31
+; GFX11-NEXT: s_and_b64 s[2:3], s[2:3], 0x50
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_and_u64_sext_with_sregs:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s2, s2, 0x50
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
+; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b64 s[2:3], s[2:3], 0x50
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
%ext = sext i32 %val to i64
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
index d1d44f7834e29..ca972118c817f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
@@ -246,15 +246,19 @@ define i64 @v_andn2_i64(i64 %src0, i64 %src1) {
; GCN-LABEL: v_andn2_i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_bfi_b32 v1, v3, 0, v1
-; GCN-NEXT: v_bfi_b32 v0, v2, 0, v0
+; GCN-NEXT: v_xor_b32_e32 v2, -1, v2
+; GCN-NEXT: v_xor_b32_e32 v3, -1, v3
+; GCN-NEXT: v_and_b32_e32 v0, v0, v2
+; GCN-NEXT: v_and_b32_e32 v1, v1, v3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_andn2_i64:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_bfi_b32 v0, v2, 0, v0
-; GFX10PLUS-NEXT: v_bfi_b32 v1, v3, 0, v1
+; GFX10PLUS-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX10PLUS-NEXT: v_xor_b32_e32 v3, -1, v3
+; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10PLUS-NEXT: v_and_b32_e32 v1, v1, v3
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%not.src1 = xor i64 %src1, -1
%and = and i64 %src0, %not.src1
@@ -264,14 +268,18 @@ define i64 @v_andn2_i64(i64 %src0, i64 %src1) {
define amdgpu_ps <2 x float> @v_andn2_i64_sv(i64 inreg %src0, i64 %src1) {
; GCN-LABEL: v_andn2_i64_sv:
; GCN: ; %bb.0:
-; GCN-NEXT: v_bfi_b32 v1, v1, 0, s3
-; GCN-NEXT: v_bfi_b32 v0, v0, 0, s2
+; GCN-NEXT: v_xor_b32_e32 v0, -1, v0
+; GCN-NEXT: v_xor_b32_e32 v1, -1, v1
+; GCN-NEXT: v_and_b32_e32 v0, s2, v0
+; GCN-NEXT: v_and_b32_e32 v1, s3, v1
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: v_andn2_i64_sv:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_bfi_b32 v0, v0, 0, s2
-; GFX10PLUS-NEXT: v_bfi_b32 v1, v1, 0, s3
+; GFX10PLUS-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX10PLUS-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX10PLUS-NEXT: v_and_b32_e32 v0, s2, v0
+; GFX10PLUS-NEXT: v_and_b32_e32 v1, s3, v1
; GFX10PLUS-NEXT: ; return to shader part epilog
%not.src1 = xor i64 %src1, -1
%and = and i64 %src0, %not.src1
@@ -282,15 +290,29 @@ define amdgpu_ps <2 x float> @v_andn2_i64_sv(i64 inreg %src0, i64 %src1) {
define amdgpu_ps <2 x float> @v_andn2_i64_vs(i64 %src0, i64 inreg %src1) {
; GCN-LABEL: v_andn2_i64_vs:
; GCN: ; %bb.0:
-; GCN-NEXT: v_bfi_b32 v1, s3, 0, v1
-; GCN-NEXT: v_bfi_b32 v0, s2, 0, v0
+; GCN-NEXT: s_not_b64 s[0:1], s[2:3]
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_and_b32_e32 v0, v0, v2
+; GCN-NEXT: v_and_b32_e32 v1, v1, v3
; GCN-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: v_andn2_i64_vs:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_bfi_b32 v0, s2, 0, v0
-; GFX10PLUS-NEXT: v_bfi_b32 v1, s3, 0, v1
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: v_andn2_i64_vs:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_not_b64 s[0:1], s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_andn2_i64_vs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_not_b64 s[0:1], s[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-NEXT: ; return to shader part epilog
%not.src1 = xor i64 %src1, -1
%and = and i64 %src0, %not.src1
%cast = bitcast i64 %and to <2 x float>
@@ -705,16 +727,25 @@ define amdgpu_ps i48 @s_andn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1
; GFX10-NEXT: s_and_b32 s1, s3, s2
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_andn2_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-NEXT: s_and_not1_b32 s0, s1, s0
-; GFX11-NEXT: s_and_not1_b32 s1, s2, s4
-; GFX11-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_and_b32 s1, s3, s2
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_andn2_v3i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-TRUE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-TRUE16-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_andn2_v3i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s2, 16
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s1, s2, s4
+; GFX11-FAKE16-NEXT: s_xor_b32 s2, s5, 0xffff
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s3, s2
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%and = and <3 x i16> %src0, %not.src1
%cast = bitcast <3 x i16> %and to i48
@@ -745,13 +776,22 @@ define amdgpu_ps i48 @s_andn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inr
; GFX10-NEXT: s_and_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_andn2_v3i16_commute:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_not1_b32 s0, s2, s4
-; GFX11-NEXT: s_xor_b32 s1, s5, 0xffff
-; GFX11-NEXT: s_pack_lh_b32_b16 s0, s0, s0
-; GFX11-NEXT: s_and_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_andn2_v3i16_commute:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-TRUE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-TRUE16-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_andn2_v3i16_commute:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s0, s2, s4
+; GFX11-FAKE16-NEXT: s_xor_b32 s1, s5, 0xffff
+; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s0, s0, s0
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, s3
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%and = and <3 x i16> %not.src1, %src0
%cast = bitcast <3 x i16> %and to i48
@@ -809,21 +849,33 @@ define amdgpu_ps { i48, i48 } @s_andn2_v3i16_multi_use(<3 x i16> inreg %src0, <3
; GFX10-NEXT: s_mov_b32 s3, s4
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_andn2_v3i16_multi_use:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-NEXT: s_not_b32 s6, s4
-; GFX11-NEXT: s_not_b32 s1, s0
-; GFX11-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX11-NEXT: s_and_not1_b32 s0, s7, s0
-; GFX11-NEXT: s_and_not1_b32 s1, s2, s4
-; GFX11-NEXT: s_not_b32 s4, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_and_not1_b32 s1, s3, s5
-; GFX11-NEXT: s_mov_b32 s2, s6
-; GFX11-NEXT: s_mov_b32 s3, s4
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_andn2_v3i16_multi_use:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-TRUE16-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX11-TRUE16-NEXT: s_and_b64 s[0:1], s[2:3], s[4:5]
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s4, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_andn2_v3i16_multi_use:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_not_b32 s1, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s2, 16
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s0, s7, s0
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s1, s2, s4
+; GFX11-FAKE16-NEXT: s_not_b32 s4, s5
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s1, s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s4
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%and = and <3 x i16> %src0, %not.src1
%cast.0 = bitcast <3 x i16> %and to i48
@@ -846,38 +898,20 @@ define <3 x i16> @v_andn2_v3i16(<3 x i16> %src0, <3 x i16> %src1) {
; GFX9-LABEL: v_andn2_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v3, -11, v3
; GFX9-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_xor_b32_e32 v3, -11, v3
; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_andn2_v3i16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX10-NEXT: v_xor_b32_e32 v3, -11, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_andn2_v3i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v3.l, -11
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_andn2_v3i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v3, -11, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX10PLUS-LABEL: v_andn2_v3i16:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10PLUS-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX10PLUS-NEXT: v_xor_b32_e32 v3, -11, v3
+; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10PLUS-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -11>
%and = and <3 x i16> %src0, %not.src1
ret <3 x i16> %and
@@ -1087,17 +1121,35 @@ define <4 x i16> @v_andn2_v4i16(<4 x i16> %src0, <4 x i16> %src1) {
; GFX6-LABEL: v_andn2_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfi_b32 v0, v2, 0, v0
-; GFX6-NEXT: v_bfi_b32 v1, v3, 0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT: v_xor_b32_e32 v3, -1, v3
+; GFX6-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v1, v1, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_andn2_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT: v_xor_b32_e32 v3, -1, v3
; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v2, -1, v3
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v2
+; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_andn2_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index 8974dfc913a25..7471343b3a9a3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -899,15 +899,24 @@ define amdgpu_ps i32 @s_ashr_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amou
;
; GFX9-LABEL: s_ashr_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, s1, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_sext_i32_i16 s2, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s3, s1
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_ashr_i32 s2, s2, s3
+; GFX9-NEXT: s_ashr_i32 s0, s0, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ashr_v2i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v0, s1, s0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: s_sext_i32_i16 s2, s0
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s3, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s2, s3
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, s1
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s2, s0
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = ashr <2 x i16> %value, %amount
%cast = bitcast <2 x i16> %result to i32
@@ -1091,20 +1100,38 @@ define amdgpu_ps <2 x i32> @s_ashr_v4i16(<4 x i16> inreg %value, <4 x i16> inreg
;
; GFX9-LABEL: s_ashr_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, s3, v0
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, s2, v1
-; GFX9-NEXT: v_readfirstlane_b32 s0, v1
-; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: s_ashr_i32 s4, s4, s5
+; GFX9-NEXT: s_ashr_i32 s0, s0, s2
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX9-NEXT: s_sext_i32_i16 s2, s1
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_sext_i32_i16 s4, s3
+; GFX9-NEXT: s_ashr_i32 s3, s3, 16
+; GFX9-NEXT: s_ashr_i32 s2, s2, s4
+; GFX9-NEXT: s_ashr_i32 s1, s1, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ashr_v4i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v0, s2, s0
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v1, s3, s1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10PLUS-NEXT: s_sext_i32_i16 s4, s0
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s5, s2
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s2, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s4, s4, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, s2
+; GFX10PLUS-NEXT: s_sext_i32_i16 s2, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s5, s3
+; GFX10PLUS-NEXT: s_ashr_i32 s3, s3, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s2, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, s3
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = ashr <4 x i16> %value, %amount
%cast = bitcast <4 x i16> %result to <2 x i32>
@@ -1286,30 +1313,66 @@ define amdgpu_ps <4 x i32> @s_ashr_v8i16(<8 x i16> inreg %value, <8 x i16> inreg
;
; GFX9-LABEL: s_ashr_v8i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, s7, v0
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, s6, v1
-; GFX9-NEXT: v_pk_ashrrev_i16 v2, s5, v2
-; GFX9-NEXT: v_pk_ashrrev_i16 v3, s4, v3
-; GFX9-NEXT: v_readfirstlane_b32 s0, v3
-; GFX9-NEXT: v_readfirstlane_b32 s1, v2
-; GFX9-NEXT: v_readfirstlane_b32 s2, v1
-; GFX9-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-NEXT: s_sext_i32_i16 s8, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s9, s4
+; GFX9-NEXT: s_ashr_i32 s4, s4, 16
+; GFX9-NEXT: s_ashr_i32 s8, s8, s9
+; GFX9-NEXT: s_ashr_i32 s0, s0, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s8, s0
+; GFX9-NEXT: s_sext_i32_i16 s4, s1
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_sext_i32_i16 s8, s5
+; GFX9-NEXT: s_ashr_i32 s5, s5, 16
+; GFX9-NEXT: s_ashr_i32 s4, s4, s8
+; GFX9-NEXT: s_ashr_i32 s1, s1, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s4, s1
+; GFX9-NEXT: s_sext_i32_i16 s4, s2
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: s_sext_i32_i16 s5, s6
+; GFX9-NEXT: s_ashr_i32 s6, s6, 16
+; GFX9-NEXT: s_ashr_i32 s4, s4, s5
+; GFX9-NEXT: s_ashr_i32 s2, s2, s6
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT: s_sext_i32_i16 s4, s3
+; GFX9-NEXT: s_ashr_i32 s3, s3, 16
+; GFX9-NEXT: s_sext_i32_i16 s5, s7
+; GFX9-NEXT: s_ashr_i32 s6, s7, 16
+; GFX9-NEXT: s_ashr_i32 s4, s4, s5
+; GFX9-NEXT: s_ashr_i32 s3, s3, s6
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s4, s3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ashr_v8i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v0, s4, s0
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v1, s5, s1
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v2, s6, s2
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v3, s7, s3
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v2
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10PLUS-NEXT: s_sext_i32_i16 s8, s0
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s9, s4
+; GFX10PLUS-NEXT: s_ashr_i32 s4, s4, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s8, s8, s9
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, s4
+; GFX10PLUS-NEXT: s_sext_i32_i16 s4, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s9, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s5, s5, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s4, s4, s9
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, s5
+; GFX10PLUS-NEXT: s_sext_i32_i16 s5, s6
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s4, s1
+; GFX10PLUS-NEXT: s_sext_i32_i16 s4, s2
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s2, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s6, s6, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s4, s4, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s2, s6
+; GFX10PLUS-NEXT: s_sext_i32_i16 s5, s3
+; GFX10PLUS-NEXT: s_ashr_i32 s3, s3, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s6, s7
+; GFX10PLUS-NEXT: s_ashr_i32 s7, s7, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s5, s5, s6
+; GFX10PLUS-NEXT: s_ashr_i32 s3, s3, s7
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s8, s0
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s3, s5, s3
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = ashr <8 x i16> %value, %amount
%cast = bitcast <8 x i16> %result to <4 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
index c0241876cbc2a..ca073d7653354 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
@@ -453,33 +453,33 @@ define amdgpu_kernel void @load_i8_to_f32(ptr addrspace(1) noalias %out, ptr add
; SI-LABEL: load_i8_to_f32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_mov_b32 s10, 0
-; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
-; SI-NEXT: buffer_load_ubyte v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: load_i8_to_f32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_ubyte v2, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s2
+; VI-NEXT: v_mov_b32_e32 v2, s3
+; VI-NEXT: v_add_u32_e32 v0, vcc, v1, v0
+; VI-NEXT: v_addc_u32_e32 v1, vcc, v2, v3, vcc
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v2
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1061,33 +1061,33 @@ define amdgpu_kernel void @i8_zext_i32_to_f32(ptr addrspace(1) noalias %out, ptr
; SI-LABEL: i8_zext_i32_to_f32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_mov_b32 s10, 0
-; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
-; SI-NEXT: buffer_load_ubyte v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: i8_zext_i32_to_f32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_ubyte v2, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s2
+; VI-NEXT: v_mov_b32_e32 v2, s3
+; VI-NEXT: v_add_u32_e32 v0, vcc, v1, v0
+; VI-NEXT: v_addc_u32_e32 v1, vcc, v2, v3, vcc
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v2
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
index cc70226e2ab72..6a43cba0f9ec5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
; This file contains various tests that have divergent i1s used outside of
; the loop. These are lane masks is sgpr and need to have correct value in
@@ -13,25 +13,22 @@ define void @divergent_i1_phi_used_outside_loop(float %val, float %pre.cond.val,
; GFX10-LABEL: divergent_i1_phi_used_outside_loop:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, 1.0, v1
+; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
-; GFX10-NEXT: s_and_b32 s5, vcc_lo, exec_lo
; GFX10-NEXT: .LBB0_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX10-NEXT: s_xor_b32 s8, s5, -1
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: s_xor_b32 s8, s5, s8
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 s9, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s8, s8, exec_lo
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX10-NEXT: s_or_b32 s8, s9, s8
-; GFX10-NEXT: s_or_b32 s7, s7, s5
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s5
; GFX10-NEXT: s_mov_b32 s5, s8
+; GFX10-NEXT: s_or_b32 s7, s7, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2: ; %exit
@@ -143,18 +140,20 @@ define void @divergent_i1_xor_used_outside_loop(float %val, float %pre.cond.val,
; GFX10-NEXT: .LBB2_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX10-NEXT: s_xor_b32 s5, s5, -1
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: s_xor_b32 s8, s5, s8
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s8, s5, exec_lo
-; GFX10-NEXT: s_or_b32 s7, s7, s8
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s5
+; GFX10-NEXT: s_mov_b32 s5, s8
+; GFX10-NEXT: s_or_b32 s7, s7, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2: ; %exit
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s7
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -183,24 +182,26 @@ define void @divergent_i1_xor_used_outside_loop_twice(float %val, float %pre.con
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v1
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_mov_b32 s7, 0
-; GFX10-NEXT: ; implicit-def: $sgpr6
+; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: .LBB3_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s7
-; GFX10-NEXT: s_xor_b32 s5, s5, -1
-; GFX10-NEXT: s_add_i32 s7, s7, 1
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: s_xor_b32 s8, s5, s8
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_and_b32 s8, s5, exec_lo
-; GFX10-NEXT: s_or_b32 s6, s6, s8
+; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, s5
+; GFX10-NEXT: s_mov_b32 s5, s8
+; GFX10-NEXT: s_or_b32 s7, s7, s9
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
; GFX10-NEXT: ; %bb.2: ; %exit
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s6
-; GFX10-NEXT: v_cndmask_b32_e64 v1, -1.0, 2.0, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 2.0, -1.0, s7
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: flat_store_dword v[4:5], v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -240,35 +241,41 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT: s_mov_b32 s6, exec_lo
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_mov_b32 s5, -1
-; GFX10-NEXT: s_and_saveexec_b32 s6, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s7, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_6
; GFX10-NEXT: ; %bb.1: ; %loop.start.preheader
-; GFX10-NEXT: s_mov_b32 s7, 0
-; GFX10-NEXT: ; implicit-def: $sgpr8
+; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: ; implicit-def: $sgpr10
+; GFX10-NEXT: ; implicit-def: $sgpr11
; GFX10-NEXT: ; implicit-def: $sgpr9
; GFX10-NEXT: s_branch .LBB4_3
; GFX10-NEXT: .LBB4_2: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB4_3 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_xor_b32 s5, s9, -1
-; GFX10-NEXT: s_and_b32 s11, exec_lo, s10
-; GFX10-NEXT: s_or_b32 s7, s11, s7
-; GFX10-NEXT: s_andn2_b32 s8, s8, exec_lo
-; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX10-NEXT: s_or_b32 s8, s8, s5
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT: s_xor_b32 s5, s11, exec_lo
+; GFX10-NEXT: s_and_b32 s12, exec_lo, s10
+; GFX10-NEXT: s_or_b32 s8, s12, s8
+; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s5
+; GFX10-NEXT: s_or_b32 s9, s9, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execz .LBB4_5
; GFX10-NEXT: .LBB4_3: ; %loop.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s5, s4, 31
-; GFX10-NEXT: s_or_b32 s9, s9, exec_lo
+; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
; GFX10-NEXT: s_lshl_b64 s[12:13], s[4:5], 2
-; GFX10-NEXT: s_or_b32 s10, s10, exec_lo
-; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, s12
-; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s13, v2, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s5, s11, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v5, s12
+; GFX10-NEXT: v_mov_b32_e32 v6, s13
+; GFX10-NEXT: s_and_b32 s11, exec_lo, exec_lo
+; GFX10-NEXT: s_and_b32 s12, exec_lo, exec_lo
+; GFX10-NEXT: s_or_b32 s11, s5, s11
+; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, v5
+; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
+; GFX10-NEXT: s_or_b32 s10, s10, s12
; GFX10-NEXT: global_load_dword v5, v[5:6], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v5
@@ -277,18 +284,22 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
; GFX10-NEXT: ; %bb.4: ; %loop.cond
; GFX10-NEXT: ; in Loop: Header=BB4_3 Depth=1
; GFX10-NEXT: v_cmp_lt_i32_e32 vcc_lo, s4, v0
+; GFX10-NEXT: s_andn2_b32 s11, s11, exec_lo
+; GFX10-NEXT: s_and_b32 s12, exec_lo, 0
; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
; GFX10-NEXT: s_add_i32 s4, s4, 1
-; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s11, vcc_lo, exec_lo
-; GFX10-NEXT: s_or_b32 s10, s10, s11
+; GFX10-NEXT: s_and_b32 s13, exec_lo, vcc_lo
+; GFX10-NEXT: s_or_b32 s11, s11, s12
+; GFX10-NEXT: s_or_b32 s10, s10, s13
; GFX10-NEXT: s_branch .LBB4_2
; GFX10-NEXT: .LBB4_5: ; %loop.exit.guard
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
-; GFX10-NEXT: s_orn2_b32 s5, s8, exec_lo
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_andn2_b32 s4, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s9
+; GFX10-NEXT: s_or_b32 s6, s4, s5
; GFX10-NEXT: .LBB4_6: ; %Flow1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: s_and_saveexec_b32 s4, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT: s_and_saveexec_b32 s4, s6
; GFX10-NEXT: s_cbranch_execz .LBB4_8
; GFX10-NEXT: ; %bb.7: ; %block.after.loop
; GFX10-NEXT: v_mov_b32_e32 v0, 5
@@ -336,52 +347,54 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace
; GFX10-LABEL: divergent_i1_icmp_used_outside_loop:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s6, 0
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: ; implicit-def: $sgpr8
+; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: s_branch .LBB5_2
; GFX10-NEXT: .LBB5_1: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX10-NEXT: s_and_b32 s4, exec_lo, s9
-; GFX10-NEXT: v_mov_b32_e32 v4, s6
-; GFX10-NEXT: s_or_b32 s5, s4, s5
-; GFX10-NEXT: s_andn2_b32 s4, s8, exec_lo
-; GFX10-NEXT: s_and_b32 s6, vcc_lo, exec_lo
-; GFX10-NEXT: s_or_b32 s8, s4, s6
-; GFX10-NEXT: s_mov_b32 s6, s7
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s5
+; GFX10-NEXT: s_or_b32 s6, s5, s6
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: s_cbranch_execz .LBB5_6
; GFX10-NEXT: .LBB5_2: ; %cond.block.0
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, s6, v0
-; GFX10-NEXT: s_and_saveexec_b32 s9, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, s4, v0
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: s_andn2_b32 s5, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, vcc_lo
+; GFX10-NEXT: s_or_b32 s7, s5, s7
+; GFX10-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_4
; GFX10-NEXT: ; %bb.3: ; %if.block.0
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT: s_ashr_i32 s7, s6, 31
-; GFX10-NEXT: v_mov_b32_e32 v8, s6
-; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
-; GFX10-NEXT: v_add_co_u32 v4, s4, v2, s10
-; GFX10-NEXT: v_add_co_ci_u32_e64 v5, s4, s11, v3, s4
-; GFX10-NEXT: global_store_dword v[4:5], v8, off
+; GFX10-NEXT: s_ashr_i32 s5, s4, 31
+; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[4:5], 2
+; GFX10-NEXT: v_mov_b32_e32 v8, s10
+; GFX10-NEXT: v_mov_b32_e32 v9, s11
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, v8
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
+; GFX10-NEXT: global_store_dword v[8:9], v5, off
; GFX10-NEXT: .LBB5_4: ; %loop.break.block
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT: v_cmp_ne_u32_e64 s4, s6, v1
-; GFX10-NEXT: s_mov_b32 s9, -1
-; GFX10-NEXT: ; implicit-def: $sgpr7
-; GFX10-NEXT: s_and_saveexec_b32 s10, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v1
+; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_1
; GFX10-NEXT: ; %bb.5: ; %loop.cond
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT: s_add_i32 s7, s6, 1
-; GFX10-NEXT: s_xor_b32 s9, exec_lo, -1
+; GFX10-NEXT: s_andn2_b32 s5, s5, exec_lo
+; GFX10-NEXT: s_and_b32 s9, exec_lo, 0
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_or_b32 s5, s5, s9
; GFX10-NEXT: s_branch .LBB5_1
; GFX10-NEXT: .LBB5_6: ; %cond.block.1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_and_saveexec_b32 s4, s8
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_and_saveexec_b32 s4, s7
; GFX10-NEXT: s_cbranch_execz .LBB5_8
; GFX10-NEXT: ; %bb.7: ; %if.block.1
; GFX10-NEXT: global_store_dword v[6:7], v4, off
@@ -445,11 +458,11 @@ exit:
define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspace(1) %a, ptr %addr) {
; GFX10-LABEL: divergent_i1_freeze_used_outside_loop:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_mov_b32 s1, -1
+; GFX10-NEXT: s_mov_b32 s1, exec_lo
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s2, 0
-; GFX10-NEXT: ; implicit-def: $sgpr3
; GFX10-NEXT: ; implicit-def: $sgpr4
+; GFX10-NEXT: ; implicit-def: $sgpr3
; GFX10-NEXT: s_branch .LBB6_2
; GFX10-NEXT: .LBB6_1: ; %loop.cond
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
@@ -457,17 +470,17 @@ define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspa
; GFX10-NEXT: v_cmp_lt_i32_e32 vcc_lo, s0, v0
; GFX10-NEXT: s_add_i32 s0, s0, 1
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX10-NEXT: s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT: s_and_b32 s5, s4, exec_lo
; GFX10-NEXT: s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT: s_or_b32 s1, s1, s5
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 s1, s1, exec_lo
; GFX10-NEXT: s_or_b32 s3, s3, s5
+; GFX10-NEXT: s_or_b32 s1, s1, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execz .LBB6_4
; GFX10-NEXT: .LBB6_2: ; %loop.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
-; GFX10-NEXT: s_and_b32 s5, s1, exec_lo
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s1
; GFX10-NEXT: s_or_b32 s4, s4, s5
; GFX10-NEXT: s_and_saveexec_b32 s5, s1
; GFX10-NEXT: s_cbranch_execz .LBB6_1
@@ -476,12 +489,14 @@ define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspa
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
; GFX10-NEXT: s_lshl_b64 s[6:7], s[0:1], 2
; GFX10-NEXT: s_andn2_b32 s1, s4, exec_lo
-; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, s6
-; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s7, v2, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v5, s6
+; GFX10-NEXT: v_mov_b32_e32 v6, s7
+; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, v5
+; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
; GFX10-NEXT: global_load_dword v5, v[5:6], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX10-NEXT: s_and_b32 s4, exec_lo, vcc_lo
; GFX10-NEXT: s_or_b32 s4, s1, s4
; GFX10-NEXT: ; implicit-def: $sgpr1
; GFX10-NEXT: s_branch .LBB6_1
@@ -523,29 +538,36 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: ; implicit-def: $sgpr5
-; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: ; implicit-def: $sgpr6
+; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr5
; GFX10-NEXT: s_branch .LBB7_2
; GFX10-NEXT: .LBB7_1: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB7_2 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX10-NEXT: s_and_b32 s1, exec_lo, s7
+; GFX10-NEXT: s_and_b32 s1, exec_lo, s6
; GFX10-NEXT: s_or_b32 s4, s1, s4
; GFX10-NEXT: s_andn2_b32 s1, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s2, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s2, exec_lo, s7
; GFX10-NEXT: s_or_b32 s5, s1, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execz .LBB7_4
; GFX10-NEXT: .LBB7_2: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_or_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 2
-; GFX10-NEXT: s_or_b32 s7, s7, exec_lo
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s1, s7, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v6, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s3
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s8
+; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s8, exec_lo, exec_lo
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT: s_or_b32 s7, s1, s7
+; GFX10-NEXT: s_or_b32 s6, s6, s8
; GFX10-NEXT: global_load_dword v6, v[6:7], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
@@ -553,16 +575,20 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10-NEXT: s_cbranch_execz .LBB7_1
; GFX10-NEXT: ; %bb.3: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB7_2 Depth=1
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v1, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v6, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s3
; GFX10-NEXT: s_add_i32 s2, s0, 1
; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: global_load_dword v8, v[6:7], off
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
; GFX10-NEXT: s_andn2_b32 s3, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, 0
; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_or_b32 s7, s3, s0
+; GFX10-NEXT: global_load_dword v8, v[6:7], off
+; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT: s_or_b32 s7, s3, s7
+; GFX10-NEXT: s_or_b32 s6, s6, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v8
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
index ccae76b6cb5c8..66af63e968b0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
; Simples case, if - then, that requires lane mask merging,
; %phi lane mask will hold %val_A at %A. Lanes that are active in %B
@@ -121,10 +121,14 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10-NEXT: .LBB2_2: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_or_b32 s5, s5, exec_lo
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 2
-; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v2, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: s_and_b32 s5, exec_lo, exec_lo
+; GFX10-NEXT: s_or_b32 s5, s1, s5
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v2, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v3, v5, vcc_lo
; GFX10-NEXT: global_load_dword v4, v[4:5], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
@@ -132,14 +136,16 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
; GFX10-NEXT: s_cbranch_execz .LBB2_1
; GFX10-NEXT: ; %bb.3: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s3, v1, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
; GFX10-NEXT: s_add_i32 s2, s0, 1
; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: global_load_dword v6, v[4:5], off
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo
; GFX10-NEXT: s_andn2_b32 s3, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT: global_load_dword v6, v[4:5], off
; GFX10-NEXT: s_or_b32 s5, s3, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -181,9 +187,9 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: .LBB3_1: ; %Flow3
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
; GFX10-NEXT: s_andn2_b32 s2, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s3, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s6
; GFX10-NEXT: s_or_b32 s5, s2, s3
; GFX10-NEXT: .LBB3_2: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
@@ -195,10 +201,14 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: .LBB3_3: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_or_b32 s5, s5, exec_lo
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 2
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v6, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s3
+; GFX10-NEXT: s_and_b32 s5, exec_lo, exec_lo
+; GFX10-NEXT: s_or_b32 s5, s1, s5
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
; GFX10-NEXT: global_load_dword v6, v[6:7], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
@@ -206,23 +216,29 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: s_cbranch_execz .LBB3_2
; GFX10-NEXT: ; %bb.4: ; %B
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v4, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v5, vcc_lo
-; GFX10-NEXT: s_mov_b32 s7, -1
+; GFX10-NEXT: v_mov_b32_e32 v6, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s3
+; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v4, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v5, v7, vcc_lo
; GFX10-NEXT: global_load_dword v6, v[6:7], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
-; GFX10-NEXT: s_and_saveexec_b32 s6, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s7, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB3_1
; GFX10-NEXT: ; %bb.5: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v1, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v6, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s3
; GFX10-NEXT: s_add_i32 s2, s0, 1
; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s3, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
; GFX10-NEXT: global_load_dword v8, v[6:7], off
-; GFX10-NEXT: s_orn2_b32 s7, s0, exec_lo
+; GFX10-NEXT: s_or_b32 s6, s3, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v8
@@ -269,13 +285,15 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: .LBB4_1: ; %Flow5
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
-; GFX10-NEXT: s_orn2_b32 s7, s8, exec_lo
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX10-NEXT: s_andn2_b32 s2, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s8
+; GFX10-NEXT: s_or_b32 s6, s2, s3
; GFX10-NEXT: .LBB4_2: ; %Flow4
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7
; GFX10-NEXT: s_andn2_b32 s2, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s3, s7, exec_lo
+; GFX10-NEXT: s_and_b32 s3, exec_lo, s6
; GFX10-NEXT: s_or_b32 s5, s2, s3
; GFX10-NEXT: .LBB4_3: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
@@ -287,10 +305,14 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: .LBB4_4: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_or_b32 s5, s5, exec_lo
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 2
-; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v8, s2
+; GFX10-NEXT: v_mov_b32_e32 v9, s3
+; GFX10-NEXT: s_and_b32 s5, exec_lo, exec_lo
+; GFX10-NEXT: s_or_b32 s5, s1, s5
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, v8
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
; GFX10-NEXT: global_load_dword v8, v[8:9], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
@@ -298,33 +320,41 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %
; GFX10-NEXT: s_cbranch_execz .LBB4_3
; GFX10-NEXT: ; %bb.5: ; %B
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v4, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, s3, v5, vcc_lo
-; GFX10-NEXT: s_mov_b32 s7, -1
+; GFX10-NEXT: v_mov_b32_e32 v8, s2
+; GFX10-NEXT: v_mov_b32_e32 v9, s3
+; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v4, v8
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v5, v9, vcc_lo
; GFX10-NEXT: global_load_dword v8, v[8:9], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX10-NEXT: s_and_saveexec_b32 s6, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s7, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_2
; GFX10-NEXT: ; %bb.6: ; %C
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v6, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, s3, v7, vcc_lo
-; GFX10-NEXT: s_mov_b32 s8, -1
+; GFX10-NEXT: v_mov_b32_e32 v8, s2
+; GFX10-NEXT: v_mov_b32_e32 v9, s3
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v6, v8
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v7, v9, vcc_lo
; GFX10-NEXT: global_load_dword v8, v[8:9], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX10-NEXT: s_and_saveexec_b32 s7, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s9, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_1
; GFX10-NEXT: ; %bb.7: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v0, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, s3, v1, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v8, s2
+; GFX10-NEXT: v_mov_b32_e32 v9, s3
; GFX10-NEXT: s_add_i32 s2, s0, 1
; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v0, v8
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v9, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s3, s8, exec_lo
+; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
; GFX10-NEXT: global_load_dword v10, v[8:9], off
-; GFX10-NEXT: s_orn2_b32 s8, s0, exec_lo
+; GFX10-NEXT: s_or_b32 s8, s3, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
@@ -376,29 +406,36 @@ define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr ad
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_mov_b32 s0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: ; implicit-def: $sgpr5
-; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: ; implicit-def: $sgpr6
+; GFX10-NEXT: ; implicit-def: $sgpr7
+; GFX10-NEXT: ; implicit-def: $sgpr5
; GFX10-NEXT: s_branch .LBB5_2
; GFX10-NEXT: .LBB5_1: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX10-NEXT: s_and_b32 s1, exec_lo, s7
+; GFX10-NEXT: s_and_b32 s1, exec_lo, s6
; GFX10-NEXT: s_or_b32 s4, s1, s4
; GFX10-NEXT: s_andn2_b32 s1, s5, exec_lo
-; GFX10-NEXT: s_and_b32 s2, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s2, exec_lo, s7
; GFX10-NEXT: s_or_b32 s5, s1, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execz .LBB5_4
; GFX10-NEXT: .LBB5_2: ; %A
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_or_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 2
-; GFX10-NEXT: s_or_b32 s7, s7, exec_lo
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: s_andn2_b32 s1, s7, exec_lo
+; GFX10-NEXT: v_mov_b32_e32 v6, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s3
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s8
+; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_and_b32 s8, exec_lo, exec_lo
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT: s_or_b32 s7, s1, s7
+; GFX10-NEXT: s_or_b32 s6, s6, s8
; GFX10-NEXT: global_load_dword v6, v[6:7], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
@@ -406,16 +443,20 @@ define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr ad
; GFX10-NEXT: s_cbranch_execz .LBB5_1
; GFX10-NEXT: ; %bb.3: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, s2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v1, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v6, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s3
; GFX10-NEXT: s_add_i32 s2, s0, 1
; GFX10-NEXT: s_cmpk_lt_u32 s0, 0x64
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: global_load_dword v8, v[6:7], off
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v0, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
; GFX10-NEXT: s_andn2_b32 s3, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, 0
; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_or_b32 s7, s3, s0
+; GFX10-NEXT: global_load_dword v8, v[6:7], off
+; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT: s_or_b32 s7, s3, s7
+; GFX10-NEXT: s_or_b32 s6, s6, s0
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v8
@@ -482,64 +523,71 @@ define amdgpu_ps i32 @irreducible_cfg(i32 %x, i32 %y, i32 %a0, i32 %a1, i32 %a2,
; GFX10-LABEL: irreducible_cfg:
; GFX10: ; %bb.0: ; %.entry
; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, v4, v1
-; GFX10-NEXT: v_cmp_le_i32_e64 s4, v4, v1
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: ; implicit-def: $sgpr3
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: s_mov_b32 s1, 0
+; GFX10-NEXT: s_and_b32 s2, s0, 1
+; GFX10-NEXT: s_xor_b32 s0, vcc_lo, s0
+; GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GFX10-NEXT: ; implicit-def: $sgpr2
-; GFX10-NEXT: s_xor_b32 s1, vcc_lo, -1
+; GFX10-NEXT: s_cselect_b32 s3, exec_lo, 0
; GFX10-NEXT: s_branch .LBB6_2
; GFX10-NEXT: .LBB6_1: ; %Flow2
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT: s_and_b32 s5, exec_lo, s7
-; GFX10-NEXT: s_or_b32 s0, s5, s0
-; GFX10-NEXT: s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT: s_and_b32 s5, s3, exec_lo
-; GFX10-NEXT: s_or_b32 s2, s2, s5
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_and_b32 s4, exec_lo, s5
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: s_or_b32 s1, s4, s1
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: s_cbranch_execz .LBB6_8
; GFX10-NEXT: .LBB6_2: ; %irr.guard
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB6_6 Depth 2
-; GFX10-NEXT: s_andn2_b32 s5, s3, exec_lo
-; GFX10-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX10-NEXT: s_mov_b32 s6, -1
-; GFX10-NEXT: s_or_b32 s3, s5, s3
-; GFX10-NEXT: s_and_saveexec_b32 s5, s4
-; GFX10-NEXT: s_xor_b32 s4, exec_lo, s5
+; GFX10-NEXT: s_mov_b32 s4, exec_lo
+; GFX10-NEXT: s_and_saveexec_b32 s5, s0
+; GFX10-NEXT: s_xor_b32 s5, exec_lo, s5
; GFX10-NEXT: ; %bb.3: ; %.loopexit
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, v5, v0
+; GFX10-NEXT: v_cmp_gt_i32_e64 s0, v5, v0
+; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s7, exec_lo
+; GFX10-NEXT: s_xor_b32 s6, vcc_lo, s6
; GFX10-NEXT: s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT: s_and_b32 s5, vcc_lo, exec_lo
-; GFX10-NEXT: s_nor_b32 s6, vcc_lo, s1
-; GFX10-NEXT: s_or_b32 s3, s3, s5
-; GFX10-NEXT: s_orn2_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_or_b32 s6, s0, s6
+; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT: s_xor_b32 s6, s6, s7
+; GFX10-NEXT: s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_and_b32 s6, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s3, s3, s0
+; GFX10-NEXT: s_or_b32 s4, s4, s6
; GFX10-NEXT: ; %bb.4: ; %Flow1
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: s_mov_b32 s4, -1
-; GFX10-NEXT: s_mov_b32 s7, -1
-; GFX10-NEXT: s_and_saveexec_b32 s5, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 s0, s2, exec_lo
+; GFX10-NEXT: s_and_b32 s2, exec_lo, s3
+; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_or_b32 s2, s0, s2
+; GFX10-NEXT: s_and_saveexec_b32 s6, s4
; GFX10-NEXT: s_cbranch_execz .LBB6_1
; GFX10-NEXT: ; %bb.5: ; %.preheader
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: s_mov_b32 s6, 0
-; GFX10-NEXT: v_cmp_le_i32_e32 vcc_lo, v4, v0
+; GFX10-NEXT: v_cmp_le_i32_e64 s0, v4, v0
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB6_6: ; %.inner_loop
; GFX10-NEXT: ; Parent Loop BB6_2 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: s_and_b32 s7, exec_lo, vcc_lo
-; GFX10-NEXT: s_or_b32 s6, s7, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_and_b32 s7, exec_lo, s0
+; GFX10-NEXT: s_or_b32 s4, s7, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB6_6
; GFX10-NEXT: ; %bb.7: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: s_xor_b32 s7, exec_lo, -1
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 s0, s5, exec_lo
+; GFX10-NEXT: s_and_b32 s4, exec_lo, 0
+; GFX10-NEXT: s_or_b32 s5, s0, s4
; GFX10-NEXT: s_branch .LBB6_1
; GFX10-NEXT: .LBB6_8: ; %.exit
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v3, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
index 6cc26f1656af5..ec2607057ecd9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
@@ -1,25 +1,27 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
define void @temporal_divergent_i1_phi(float %val, ptr %addr) {
; GFX10-LABEL: temporal_divergent_i1_phi:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s6, -1
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: s_mov_b32 s5, 1
+; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: .LBB0_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s5
-; GFX10-NEXT: s_xor_b32 s8, s6, -1
-; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s6
+; GFX10-NEXT: s_and_b32 s8, s5, 1
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
+; GFX10-NEXT: s_xor_b32 s5, s5, 1
+; GFX10-NEXT: s_add_i32 s6, s6, 1
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s9, s6, exec_lo
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_or_b32 s7, s7, s9
+; GFX10-NEXT: s_and_b32 s8, exec_lo, s8
+; GFX10-NEXT: s_or_b32 s7, s7, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2: ; %exit
@@ -50,25 +52,28 @@ define void @temporal_divergent_i1_non_phi(float %val, ptr %addr) {
; GFX10-LABEL: temporal_divergent_i1_non_phi:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, -1
; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 1
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: ; implicit-def: $sgpr7
; GFX10-NEXT: .LBB1_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s6
-; GFX10-NEXT: s_xor_b32 s5, s5, -1
-; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: s_and_b32 s8, s5, 1
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
+; GFX10-NEXT: s_xor_b32 s5, s5, 1
+; GFX10-NEXT: s_add_i32 s6, s6, 1
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_and_b32 s8, s5, exec_lo
+; GFX10-NEXT: s_and_b32 s8, exec_lo, s8
; GFX10-NEXT: s_or_b32 s7, s7, s8
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB1_1
; GFX10-NEXT: ; %bb.2: ; %exit
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s7
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s7
; GFX10-NEXT: flat_store_dword v[1:2], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -98,29 +103,33 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: ; implicit-def: $sgpr9
; GFX10-NEXT: ; implicit-def: $sgpr10
+; GFX10-NEXT: ; implicit-def: $sgpr9
; GFX10-NEXT: s_branch .LBB2_3
; GFX10-NEXT: .LBB2_1: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, s6
-; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s7, v1, vcc_lo
-; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s4, v2
+; GFX10-NEXT: v_mov_b32_e32 v4, s6
+; GFX10-NEXT: v_mov_b32_e32 v5, s7
; GFX10-NEXT: s_andn2_b32 s6, s10, exec_lo
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo
+; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s4, v2
; GFX10-NEXT: s_add_i32 s4, s4, 1
; GFX10-NEXT: global_load_dword v6, v[4:5], off
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_and_b32 s7, vcc_lo, exec_lo
+; GFX10-NEXT: s_and_b32 s7, exec_lo, vcc_lo
; GFX10-NEXT: s_or_b32 s10, s6, s7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v6, 1, v6
; GFX10-NEXT: global_store_dword v[4:5], v6, off
; GFX10-NEXT: .LBB2_2: ; %Flow
; GFX10-NEXT: ; in Loop: Header=BB2_3 Depth=1
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
; GFX10-NEXT: s_and_b32 s6, exec_lo, s10
; GFX10-NEXT: s_or_b32 s8, s6, s8
; GFX10-NEXT: s_andn2_b32 s6, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX10-NEXT: s_and_b32 s5, exec_lo, s5
; GFX10-NEXT: s_or_b32 s9, s6, s5
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -131,13 +140,18 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: s_lshl_b64 s[6:7], s[4:5], 2
; GFX10-NEXT: s_add_u32 s12, s0, s6
; GFX10-NEXT: s_addc_u32 s13, s1, s7
-; GFX10-NEXT: s_or_b32 s10, s10, exec_lo
; GFX10-NEXT: global_load_dword v4, v3, s[12:13]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX10-NEXT: s_cbranch_vccnz .LBB2_1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v4
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT: s_and_b32 s11, exec_lo, exec_lo
+; GFX10-NEXT: s_or_b32 s10, s10, s11
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: s_cbranch_scc1 .LBB2_1
; GFX10-NEXT: ; %bb.4: ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT: s_mov_b32 s5, -1
+; GFX10-NEXT: s_mov_b32 s5, 1
; GFX10-NEXT: ; implicit-def: $sgpr4
; GFX10-NEXT: s_branch .LBB2_2
; GFX10-NEXT: .LBB2_5: ; %loop.exit.guard
@@ -146,9 +160,9 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB2_7
; GFX10-NEXT: ; %bb.6: ; %break.body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, 10
-; GFX10-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 10
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-NEXT: .LBB2_7: ; %exit
; GFX10-NEXT: s_endpgm
entry:
@@ -190,36 +204,41 @@ define void @nested_loops_temporal_divergence_inner(float %pre.cond.val, i32 %n.
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB3_2 Depth 2
; GFX10-NEXT: s_ashr_i32 s7, s6, 31
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
; GFX10-NEXT: ; implicit-def: $sgpr9
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, s10
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s11, v3, vcc_lo
-; GFX10-NEXT: s_mov_b32 s10, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s10
+; GFX10-NEXT: v_mov_b32_e32 v7, s11
+; GFX10-NEXT: s_mov_b32 s10, 0
; GFX10-NEXT: s_mov_b32 s11, 0
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[6:7]
; GFX10-NEXT: .LBB3_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB3_1 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_cvt_f32_u32_e32 v6, s11
-; GFX10-NEXT: s_xor_b32 s10, s10, -1
+; GFX10-NEXT: s_mov_b32 s12, exec_lo
; GFX10-NEXT: s_add_i32 s11, s11, 1
+; GFX10-NEXT: s_xor_b32 s4, s4, s12
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v0
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_or_b32 s10, vcc_lo, s10
; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s12, s10, exec_lo
+; GFX10-NEXT: s_and_b32 s12, exec_lo, s4
; GFX10-NEXT: s_or_b32 s9, s9, s12
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
; GFX10-NEXT: s_cbranch_execnz .LBB3_2
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB3_1 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: v_mov_b32_e32 v6, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
-; GFX10-NEXT: v_add_co_u32 v6, s4, v4, s6
-; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, s7, v5, s4
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: v_add_co_u32 v6, s4, v4, v6
+; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: flat_store_byte v[6:7], v0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -276,36 +295,41 @@ define void @nested_loops_temporal_divergence_outer(float %pre.cond.val, i32 %n.
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB4_2 Depth 2
; GFX10-NEXT: s_ashr_i32 s7, s6, 31
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
; GFX10-NEXT: ; implicit-def: $sgpr9
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, s10
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s11, v3, vcc_lo
-; GFX10-NEXT: s_mov_b32 s10, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s10
+; GFX10-NEXT: v_mov_b32_e32 v7, s11
+; GFX10-NEXT: s_mov_b32 s10, 0
; GFX10-NEXT: s_mov_b32 s11, 0
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[6:7]
; GFX10-NEXT: .LBB4_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB4_1 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_cvt_f32_u32_e32 v6, s11
-; GFX10-NEXT: s_xor_b32 s10, s10, -1
+; GFX10-NEXT: s_mov_b32 s12, exec_lo
; GFX10-NEXT: s_add_i32 s11, s11, 1
+; GFX10-NEXT: s_xor_b32 s4, s4, s12
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v0
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_or_b32 s10, vcc_lo, s10
; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s12, s10, exec_lo
+; GFX10-NEXT: s_and_b32 s12, exec_lo, s4
; GFX10-NEXT: s_or_b32 s9, s9, s12
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
; GFX10-NEXT: s_cbranch_execnz .LBB4_2
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB4_1 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: v_mov_b32_e32 v6, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
-; GFX10-NEXT: v_add_co_u32 v6, s4, v4, s6
-; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, s7, v5, s4
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: v_add_co_u32 v6, s4, v4, v6
+; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: flat_store_byte v[6:7], v0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -359,52 +383,51 @@ define void @nested_loops_temporal_divergence_both(float %pre.cond.val, i32 %n.i
; GFX10-NEXT: s_mov_b32 s6, 0
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: ; implicit-def: $sgpr9
-; GFX10-NEXT: ; implicit-def: $sgpr10
; GFX10-NEXT: .LBB5_1: ; %OuterHeader
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB5_2 Depth 2
; GFX10-NEXT: s_ashr_i32 s7, s6, 31
-; GFX10-NEXT: s_mov_b32 s11, s5
-; GFX10-NEXT: s_lshl_b64 s[12:13], s[6:7], 2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, s12
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, s13, v3, vcc_lo
-; GFX10-NEXT: s_mov_b32 s12, 0
+; GFX10-NEXT: s_mov_b32 s4, s5
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
+; GFX10-NEXT: v_mov_b32_e32 v8, s10
+; GFX10-NEXT: v_mov_b32_e32 v9, s11
+; GFX10-NEXT: s_mov_b32 s10, 0
+; GFX10-NEXT: s_mov_b32 s11, 0
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, v8
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[8:9]
; GFX10-NEXT: .LBB5_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB5_1 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_cvt_f32_u32_e32 v8, s12
-; GFX10-NEXT: s_xor_b32 s11, s11, -1
-; GFX10-NEXT: s_add_i32 s12, s12, 1
+; GFX10-NEXT: v_cvt_f32_u32_e32 v8, s11
+; GFX10-NEXT: s_mov_b32 s12, exec_lo
+; GFX10-NEXT: s_add_i32 s11, s11, 1
+; GFX10-NEXT: s_xor_b32 s4, s4, s12
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v0
-; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
-; GFX10-NEXT: s_and_b32 s13, s11, exec_lo
-; GFX10-NEXT: s_or_b32 s10, s10, s13
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX10-NEXT: s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT: s_and_b32 s12, exec_lo, s4
+; GFX10-NEXT: s_or_b32 s9, s9, s12
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
; GFX10-NEXT: s_cbranch_execnz .LBB5_2
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT: v_mov_b32_e32 v8, s6
+; GFX10-NEXT: v_mov_b32_e32 v9, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
-; GFX10-NEXT: v_add_co_u32 v8, s4, v4, s6
-; GFX10-NEXT: v_add_co_ci_u32_e64 v9, s4, s7, v5, s4
-; GFX10-NEXT: s_add_i32 s4, s6, 1
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s10
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
+; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: v_add_co_u32 v8, s4, v4, v8
+; GFX10-NEXT: v_add_co_ci_u32_e64 v9, s4, v5, v9, s4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
-; GFX10-NEXT: s_andn2_b32 s6, s9, exec_lo
-; GFX10-NEXT: s_and_b32 s7, s10, exec_lo
-; GFX10-NEXT: s_or_b32 s9, s6, s7
-; GFX10-NEXT: s_mov_b32 s6, s4
; GFX10-NEXT: flat_store_byte v[8:9], v0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB5_1
; GFX10-NEXT: ; %bb.4: ; %exit
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
; GFX10-NEXT: flat_store_byte v[6:7], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
index bc1dd207948c8..aaff0fc4848ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX11 %s
@gv = external addrspace(4) constant i32
@@ -8,36 +8,38 @@
define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align4(i32 %n) #0 {
; GFX9-LABEL: kernel_dynamic_stackalloc_sgpr_align4:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s4, s[8:9], 0x0
+; GFX9-NEXT: s_load_dword s5, s[8:9], 0x0
; GFX9-NEXT: s_add_u32 s0, s0, s17
-; GFX9-NEXT: s_addc_u32 s1, s1, 0
; GFX9-NEXT: s_movk_i32 s32, 0x400
-; GFX9-NEXT: s_mov_b32 s5, s32
+; GFX9-NEXT: s_addc_u32 s1, s1, 0
+; GFX9-NEXT: s_mov_b32 s4, s32
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl2_add_u32 s4, s4, 15
-; GFX9-NEXT: s_and_b32 s4, s4, -16
+; GFX9-NEXT: s_lshl2_add_u32 s5, s5, 15
+; GFX9-NEXT: s_and_b32 s5, s5, -16
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_lshl_b32 s4, s4, 6
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: s_lshl_b32 s5, s5, 6
; GFX9-NEXT: s_mov_b32 s33, 0
-; GFX9-NEXT: s_add_i32 s32, s5, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX9-NEXT: s_add_u32 s32, s4, s5
+; GFX9-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: kernel_dynamic_stackalloc_sgpr_align4:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s4, s[8:9], 0x0
+; GFX10-NEXT: s_load_dword s5, s[8:9], 0x0
+; GFX10-NEXT: s_movk_i32 s32, 0x200
; GFX10-NEXT: s_add_u32 s0, s0, s17
+; GFX10-NEXT: s_mov_b32 s4, s32
; GFX10-NEXT: s_addc_u32 s1, s1, 0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_movk_i32 s32, 0x200
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: s_mov_b32 s33, 0
-; GFX10-NEXT: s_mov_b32 s5, s32
-; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX10-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshl2_add_u32 s4, s4, 15
-; GFX10-NEXT: s_and_b32 s4, s4, -16
-; GFX10-NEXT: s_lshl_b32 s4, s4, 5
-; GFX10-NEXT: s_add_i32 s32, s5, s4
+; GFX10-NEXT: s_lshl2_add_u32 s5, s5, 15
+; GFX10-NEXT: s_and_b32 s5, s5, -16
+; GFX10-NEXT: s_lshl_b32 s5, s5, 5
+; GFX10-NEXT: s_add_u32 s32, s4, s5
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: kernel_dynamic_stackalloc_sgpr_align4:
@@ -63,50 +65,52 @@ define void @func_dynamic_stackalloc_sgpr_align4() #0 {
; GFX9-LABEL: func_dynamic_stackalloc_sgpr_align4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s6, s33
+; GFX9-NEXT: s_mov_b32 s7, s33
; GFX9-NEXT: s_mov_b32 s33, s32
; GFX9-NEXT: s_addk_i32 s32, 0x400
; GFX9-NEXT: s_getpc_b64 s[4:5]
; GFX9-NEXT: s_add_u32 s4, s4, gv at gotpcrel32@lo+4
; GFX9-NEXT: s_addc_u32 s5, s5, gv at gotpcrel32@hi+12
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GFX9-NEXT: s_mov_b32 s6, s32
; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX9-NEXT: s_mov_b32 s5, s32
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s5
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX9-NEXT: s_and_b32 s4, s4, -16
; GFX9-NEXT: s_lshl_b32 s4, s4, 6
-; GFX9-NEXT: s_add_i32 s32, s5, s4
+; GFX9-NEXT: s_add_u32 s32, s6, s4
; GFX9-NEXT: s_mov_b32 s32, s33
-; GFX9-NEXT: s_mov_b32 s33, s6
+; GFX9-NEXT: s_mov_b32 s33, s7
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: func_dynamic_stackalloc_sgpr_align4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s6, s33
+; GFX10-NEXT: s_mov_b32 s7, s33
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[4:5]
; GFX10-NEXT: s_add_u32 s4, s4, gv at gotpcrel32@lo+4
; GFX10-NEXT: s_addc_u32 s5, s5, gv at gotpcrel32@hi+12
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s6, s32
; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX10-NEXT: s_mov_b32 s5, s32
-; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s5
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX10-NEXT: s_and_b32 s4, s4, -16
; GFX10-NEXT: s_lshl_b32 s4, s4, 5
-; GFX10-NEXT: s_add_i32 s32, s5, s4
+; GFX10-NEXT: s_add_u32 s32, s6, s4
; GFX10-NEXT: s_mov_b32 s32, s33
-; GFX10-NEXT: s_mov_b32 s33, s6
+; GFX10-NEXT: s_mov_b32 s33, s7
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: func_dynamic_stackalloc_sgpr_align4:
@@ -141,36 +145,38 @@ define void @func_dynamic_stackalloc_sgpr_align4() #0 {
define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align16(i32 %n) #0 {
; GFX9-LABEL: kernel_dynamic_stackalloc_sgpr_align16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s4, s[8:9], 0x0
+; GFX9-NEXT: s_load_dword s5, s[8:9], 0x0
; GFX9-NEXT: s_add_u32 s0, s0, s17
-; GFX9-NEXT: s_addc_u32 s1, s1, 0
; GFX9-NEXT: s_movk_i32 s32, 0x400
-; GFX9-NEXT: s_mov_b32 s5, s32
+; GFX9-NEXT: s_addc_u32 s1, s1, 0
+; GFX9-NEXT: s_mov_b32 s4, s32
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl2_add_u32 s4, s4, 15
-; GFX9-NEXT: s_and_b32 s4, s4, -16
+; GFX9-NEXT: s_lshl2_add_u32 s5, s5, 15
+; GFX9-NEXT: s_and_b32 s5, s5, -16
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_lshl_b32 s4, s4, 6
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: s_lshl_b32 s5, s5, 6
; GFX9-NEXT: s_mov_b32 s33, 0
-; GFX9-NEXT: s_add_i32 s32, s5, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX9-NEXT: s_add_u32 s32, s4, s5
+; GFX9-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: kernel_dynamic_stackalloc_sgpr_align16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s4, s[8:9], 0x0
+; GFX10-NEXT: s_load_dword s5, s[8:9], 0x0
+; GFX10-NEXT: s_movk_i32 s32, 0x200
; GFX10-NEXT: s_add_u32 s0, s0, s17
+; GFX10-NEXT: s_mov_b32 s4, s32
; GFX10-NEXT: s_addc_u32 s1, s1, 0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_movk_i32 s32, 0x200
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: s_mov_b32 s33, 0
-; GFX10-NEXT: s_mov_b32 s5, s32
-; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX10-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshl2_add_u32 s4, s4, 15
-; GFX10-NEXT: s_and_b32 s4, s4, -16
-; GFX10-NEXT: s_lshl_b32 s4, s4, 5
-; GFX10-NEXT: s_add_i32 s32, s5, s4
+; GFX10-NEXT: s_lshl2_add_u32 s5, s5, 15
+; GFX10-NEXT: s_and_b32 s5, s5, -16
+; GFX10-NEXT: s_lshl_b32 s5, s5, 5
+; GFX10-NEXT: s_add_u32 s32, s4, s5
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: kernel_dynamic_stackalloc_sgpr_align16:
@@ -196,50 +202,52 @@ define void @func_dynamic_stackalloc_sgpr_align16() #0 {
; GFX9-LABEL: func_dynamic_stackalloc_sgpr_align16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s6, s33
+; GFX9-NEXT: s_mov_b32 s7, s33
; GFX9-NEXT: s_mov_b32 s33, s32
; GFX9-NEXT: s_addk_i32 s32, 0x400
; GFX9-NEXT: s_getpc_b64 s[4:5]
; GFX9-NEXT: s_add_u32 s4, s4, gv at gotpcrel32@lo+4
; GFX9-NEXT: s_addc_u32 s5, s5, gv at gotpcrel32@hi+12
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GFX9-NEXT: s_mov_b32 s6, s32
; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX9-NEXT: s_mov_b32 s5, s32
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s5
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX9-NEXT: s_and_b32 s4, s4, -16
; GFX9-NEXT: s_lshl_b32 s4, s4, 6
-; GFX9-NEXT: s_add_i32 s32, s5, s4
+; GFX9-NEXT: s_add_u32 s32, s6, s4
; GFX9-NEXT: s_mov_b32 s32, s33
-; GFX9-NEXT: s_mov_b32 s33, s6
+; GFX9-NEXT: s_mov_b32 s33, s7
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: func_dynamic_stackalloc_sgpr_align16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s6, s33
+; GFX10-NEXT: s_mov_b32 s7, s33
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[4:5]
; GFX10-NEXT: s_add_u32 s4, s4, gv at gotpcrel32@lo+4
; GFX10-NEXT: s_addc_u32 s5, s5, gv at gotpcrel32@hi+12
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s6, s32
; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX10-NEXT: s_mov_b32 s5, s32
-; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s5
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX10-NEXT: s_and_b32 s4, s4, -16
; GFX10-NEXT: s_lshl_b32 s4, s4, 5
-; GFX10-NEXT: s_add_i32 s32, s5, s4
+; GFX10-NEXT: s_add_u32 s32, s6, s4
; GFX10-NEXT: s_mov_b32 s32, s33
-; GFX10-NEXT: s_mov_b32 s33, s6
+; GFX10-NEXT: s_mov_b32 s33, s7
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: func_dynamic_stackalloc_sgpr_align16:
@@ -278,16 +286,16 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align32(i32 %n) #0 {
; GFX9-NEXT: s_movk_i32 s32, 0x800
; GFX9-NEXT: s_add_u32 s0, s0, s17
; GFX9-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-NEXT: s_add_i32 s5, s32, 0x7ff
+; GFX9-NEXT: s_add_u32 s5, s32, 0x7ff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX9-NEXT: s_and_b32 s5, s5, 0xfffff800
; GFX9-NEXT: s_and_b32 s4, s4, -16
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_lshl_b32 s4, s4, 6
; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: s_lshl_b32 s4, s4, 6
; GFX9-NEXT: s_mov_b32 s33, 0
-; GFX9-NEXT: s_add_i32 s32, s5, s4
+; GFX9-NEXT: s_add_u32 s32, s5, s4
; GFX9-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-NEXT: s_endpgm
;
@@ -297,7 +305,7 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align32(i32 %n) #0 {
; GFX10-NEXT: s_movk_i32 s32, 0x400
; GFX10-NEXT: s_add_u32 s0, s0, s17
; GFX10-NEXT: s_addc_u32 s1, s1, 0
-; GFX10-NEXT: s_add_i32 s5, s32, 0x3ff
+; GFX10-NEXT: s_add_u32 s5, s32, 0x3ff
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_and_b32 s5, s5, 0xfffffc00
; GFX10-NEXT: s_mov_b32 s33, 0
@@ -307,7 +315,7 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align32(i32 %n) #0 {
; GFX10-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX10-NEXT: s_and_b32 s4, s4, -16
; GFX10-NEXT: s_lshl_b32 s4, s4, 5
-; GFX10-NEXT: s_add_i32 s32, s5, s4
+; GFX10-NEXT: s_add_u32 s32, s5, s4
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: kernel_dynamic_stackalloc_sgpr_align32:
@@ -348,7 +356,7 @@ define void @func_dynamic_stackalloc_sgpr_align32(ptr addrspace(1) %out) #0 {
; GFX9-NEXT: s_mov_b32 s33, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX9-NEXT: s_add_i32 s5, s32, 0x7ff
+; GFX9-NEXT: s_add_u32 s5, s32, 0x7ff
; GFX9-NEXT: s_and_b32 s5, s5, 0xfffff800
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
@@ -356,7 +364,7 @@ define void @func_dynamic_stackalloc_sgpr_align32(ptr addrspace(1) %out) #0 {
; GFX9-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX9-NEXT: s_and_b32 s4, s4, -16
; GFX9-NEXT: s_lshl_b32 s4, s4, 6
-; GFX9-NEXT: s_add_i32 s32, s5, s4
+; GFX9-NEXT: s_add_u32 s32, s5, s4
; GFX9-NEXT: s_mov_b32 s32, s34
; GFX9-NEXT: s_mov_b32 s34, s7
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -379,7 +387,7 @@ define void @func_dynamic_stackalloc_sgpr_align32(ptr addrspace(1) %out) #0 {
; GFX10-NEXT: s_mov_b32 s33, s6
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX10-NEXT: s_add_i32 s5, s32, 0x3ff
+; GFX10-NEXT: s_add_u32 s5, s32, 0x3ff
; GFX10-NEXT: s_and_b32 s5, s5, 0xfffffc00
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
@@ -387,7 +395,7 @@ define void @func_dynamic_stackalloc_sgpr_align32(ptr addrspace(1) %out) #0 {
; GFX10-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX10-NEXT: s_and_b32 s4, s4, -16
; GFX10-NEXT: s_lshl_b32 s4, s4, 5
-; GFX10-NEXT: s_add_i32 s32, s5, s4
+; GFX10-NEXT: s_add_u32 s32, s5, s4
; GFX10-NEXT: s_mov_b32 s32, s34
; GFX10-NEXT: s_mov_b32 s34, s7
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
index 9bed62ec89eeb..a7eb36f84531a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
@@ -1,33 +1,73 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
define amdgpu_ps i8 @extractelement_sgpr_v4i8_sgpr_idx(ptr addrspace(4) inreg %ptr, i32 inreg %idx) {
-; GCN-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s0, s[2:3], 0x0
-; GCN-NEXT: s_lshl_b32 s1, s4, 3
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshr_b32 s0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s0, s4, 3
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: s_add_u32 s0, s2, s0
+; GFX9-NEXT: s_addc_u32 s1, s3, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s0, s4, 3
+; GFX8-NEXT: s_ashr_i32 s1, s0, 31
+; GFX8-NEXT: s_add_u32 s0, s2, s0
+; GFX8-NEXT: s_addc_u32 s1, s3, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_and_b32 s4, s4, 3
+; GFX7-NEXT: s_ashr_i32 s5, s4, 31
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s3
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX10-NEXT: s_lshl_b32 s1, s4, 3
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s0, s0, s1
+; GFX10-NEXT: s_and_b32 s0, s4, 3
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_ashr_i32 s1, s0, 31
+; GFX10-NEXT: s_add_u32 s0, s2, s0
+; GFX10-NEXT: s_addc_u32 s1, s3, s1
+; GFX10-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: s_lshl_b32 s1, s4, 3
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s0, s0, s1
+; GFX11-NEXT: s_and_b32 s0, s4, 3
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_ashr_i32 s1, s0, 31
+; GFX11-NEXT: s_add_u32 s0, s2, s0
+; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: global_load_u8 v0, v0, s[0:1]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <4 x i8>, ptr addrspace(4) %ptr
%element = extractelement <4 x i8> %vector, i32 %idx
@@ -37,51 +77,65 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_sgpr_idx(ptr addrspace(4) inreg %p
define amdgpu_ps i8 @extractelement_vgpr_v4i8_sgpr_idx(ptr addrspace(1) %ptr, i32 inreg %idx) {
; GFX9-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: global_load_dword v0, v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s0, s2, 3
+; GFX9-NEXT: s_and_b32 s0, s2, 3
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s0, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v0, v[0:1]
-; GFX8-NEXT: s_lshl_b32 s0, s2, 3
+; GFX8-NEXT: s_and_b32 s0, s2, 3
+; GFX8-NEXT: s_ashr_i32 s1, s0, 31
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s0, v0
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 3
+; GFX7-NEXT: s_and_b32 s0, s2, 3
+; GFX7-NEXT: s_ashr_i32 s1, s0, 31
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s0, v0
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
-; GFX10-NEXT: s_lshl_b32 s0, s2, 3
+; GFX10-NEXT: s_and_b32 s0, s2, 3
+; GFX10-NEXT: s_ashr_i32 s1, s0, 31
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s0, v0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: s_lshl_b32 s0, s2, 3
+; GFX11-NEXT: s_and_b32 s0, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_ashr_i32 s1, s0, 31
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <4 x i8>, ptr addrspace(1) %ptr
@@ -93,51 +147,60 @@ define i8 @extractelement_vgpr_v4i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v[0:1], off
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v0, v[0:1]
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v2
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 3, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%vector = load <4 x i8>, ptr addrspace(1) %ptr
%element = extractelement <4 x i8> %vector, i32 %idx
@@ -147,47 +210,67 @@ define i8 @extractelement_vgpr_v4i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e64 v0, v0, s0
+; GFX9-NEXT: v_and_b32_e32 v2, 3, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e64 v0, v0, s0
+; GFX8-NEXT: v_and_b32_e32 v2, 3, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshr_b32_e32 v0, s0, v0
+; GFX7-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s3
+; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e64 v0, v0, s0
+; GFX10-NEXT: v_and_b32_e32 v2, 3, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e64 v0, v0, s0
+; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 3, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <4 x i8>, ptr addrspace(4) %ptr
@@ -576,28 +659,68 @@ define i8 @extractelement_vgpr_v4i8_idx3(ptr addrspace(1) %ptr) {
}
define amdgpu_ps i8 @extractelement_sgpr_v8i8_sgpr_idx(ptr addrspace(4) inreg %ptr, i32 inreg %idx) {
-; GCN-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GCN-NEXT: s_lshl_b32 s2, s4, 3
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s0, s4, 7
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: s_add_u32 s0, s2, s0
+; GFX9-NEXT: s_addc_u32 s1, s3, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s0, s4, 7
+; GFX8-NEXT: s_ashr_i32 s1, s0, 31
+; GFX8-NEXT: s_add_u32 s0, s2, s0
+; GFX8-NEXT: s_addc_u32 s1, s3, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_and_b32 s4, s4, 7
+; GFX7-NEXT: s_ashr_i32 s5, s4, 31
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s3
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: s_lshl_b32 s2, s4, 3
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT: s_and_b32 s0, s4, 7
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_ashr_i32 s1, s0, 31
+; GFX10-NEXT: s_add_u32 s0, s2, s0
+; GFX10-NEXT: s_addc_u32 s1, s3, s1
+; GFX10-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: s_lshl_b32 s2, s4, 3
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: s_and_b32 s0, s4, 7
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_ashr_i32 s1, s0, 31
+; GFX11-NEXT: s_add_u32 s0, s2, s0
+; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: global_load_u8 v0, v0, s[0:1]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <8 x i8>, ptr addrspace(4) %ptr
%element = extractelement <8 x i8> %vector, i32 %idx
@@ -607,51 +730,65 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_sgpr_idx(ptr addrspace(4) inreg %p
define amdgpu_ps i8 @extractelement_vgpr_v8i8_sgpr_idx(ptr addrspace(1) %ptr, i32 inreg %idx) {
; GFX9-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s0, s2, 3
+; GFX9-NEXT: s_and_b32 s0, s2, 7
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], s0, v[0:1]
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_lshl_b32 s0, s2, 3
+; GFX8-NEXT: s_and_b32 s0, s2, 7
+; GFX8-NEXT: s_ashr_i32 s1, s0, 31
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], s0, v[0:1]
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 3
+; GFX7-NEXT: s_and_b32 s0, s2, 7
+; GFX7-NEXT: s_ashr_i32 s1, s0, 31
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshr_b64 v[0:1], v[0:1], s0
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: s_lshl_b32 s0, s2, 3
+; GFX10-NEXT: s_and_b32 s0, s2, 7
+; GFX10-NEXT: s_ashr_i32 s1, s0, 31
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], s0, v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: s_lshl_b32 s0, s2, 3
+; GFX11-NEXT: s_and_b32 s0, s2, 7
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_ashr_i32 s1, s0, 31
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s0, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <8 x i8>, ptr addrspace(1) %ptr
@@ -663,51 +800,60 @@ define i8 @extractelement_vgpr_v8i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshr_b64 v[0:1], v[0:1], v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
%vector = load <8 x i8>, ptr addrspace(1) %ptr
%element = extractelement <8 x i8> %vector, i32 %idx
@@ -717,47 +863,67 @@ define i8 @extractelement_vgpr_v8i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
+; GFX8-NEXT: v_and_b32_e32 v2, 7, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshr_b64 v[0:1], s[0:1], v0
+; GFX7-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s3
+; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
+; GFX10-NEXT: v_and_b32_e32 v2, 7, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 7, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <8 x i8>, ptr addrspace(4) %ptr
@@ -1530,148 +1696,68 @@ define i8 @extractelement_vgpr_v8i8_idx7(ptr addrspace(1) %ptr) {
}
define amdgpu_ps i8 @extractelement_sgpr_v16i8_sgpr_idx(ptr addrspace(4) inreg %ptr, i32 inreg %idx) {
-; GCN-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshr_b32 s5, s0, 8
-; GCN-NEXT: s_cmp_eq_u32 s4, 1
-; GCN-NEXT: s_cselect_b32 s5, s5, s0
-; GCN-NEXT: s_lshr_b32 s6, s0, 16
-; GCN-NEXT: s_cmp_eq_u32 s4, 2
-; GCN-NEXT: s_cselect_b32 s5, s6, s5
-; GCN-NEXT: s_lshr_b32 s0, s0, 24
-; GCN-NEXT: s_cmp_eq_u32 s4, 3
-; GCN-NEXT: s_cselect_b32 s0, s0, s5
-; GCN-NEXT: s_cmp_eq_u32 s4, 4
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: s_lshr_b32 s5, s1, 8
-; GCN-NEXT: s_cmp_eq_u32 s4, 5
-; GCN-NEXT: s_cselect_b32 s0, s5, s0
-; GCN-NEXT: s_lshr_b32 s5, s1, 16
-; GCN-NEXT: s_cmp_eq_u32 s4, 6
-; GCN-NEXT: s_cselect_b32 s0, s5, s0
-; GCN-NEXT: s_lshr_b32 s1, s1, 24
-; GCN-NEXT: s_cmp_eq_u32 s4, 7
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: s_cmp_eq_u32 s4, 8
-; GCN-NEXT: s_cselect_b32 s0, s2, s0
-; GCN-NEXT: s_lshr_b32 s1, s2, 8
-; GCN-NEXT: s_cmp_eq_u32 s4, 9
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: s_lshr_b32 s1, s2, 16
-; GCN-NEXT: s_cmp_eq_u32 s4, 10
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: s_lshr_b32 s1, s2, 24
-; GCN-NEXT: s_cmp_eq_u32 s4, 11
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: s_cmp_eq_u32 s4, 12
-; GCN-NEXT: s_cselect_b32 s0, s3, s0
-; GCN-NEXT: s_lshr_b32 s1, s3, 8
-; GCN-NEXT: s_cmp_eq_u32 s4, 13
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: s_lshr_b32 s1, s3, 16
-; GCN-NEXT: s_cmp_eq_u32 s4, 14
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: s_lshr_b32 s1, s3, 24
-; GCN-NEXT: s_cmp_eq_u32 s4, 15
-; GCN-NEXT: s_cselect_b32 s0, s1, s0
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s0, s4, 15
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: s_add_u32 s0, s2, s0
+; GFX9-NEXT: s_addc_u32 s1, s3, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s0, s4, 15
+; GFX8-NEXT: s_ashr_i32 s1, s0, 31
+; GFX8-NEXT: s_add_u32 s0, s2, s0
+; GFX8-NEXT: s_addc_u32 s1, s3, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_and_b32 s4, s4, 15
+; GFX7-NEXT: s_ashr_i32 s5, s4, 31
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s3
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s5, s0, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 1
-; GFX10-NEXT: s_cselect_b32 s5, s5, s0
-; GFX10-NEXT: s_lshr_b32 s6, s0, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 2
-; GFX10-NEXT: s_cselect_b32 s5, s6, s5
-; GFX10-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-NEXT: s_cmp_eq_u32 s4, 3
-; GFX10-NEXT: s_cselect_b32 s0, s0, s5
-; GFX10-NEXT: s_cmp_eq_u32 s4, 4
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
-; GFX10-NEXT: s_lshr_b32 s5, s1, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 5
-; GFX10-NEXT: s_cselect_b32 s0, s5, s0
-; GFX10-NEXT: s_lshr_b32 s5, s1, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 6
-; GFX10-NEXT: s_cselect_b32 s0, s5, s0
-; GFX10-NEXT: s_lshr_b32 s1, s1, 24
-; GFX10-NEXT: s_cmp_eq_u32 s4, 7
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 8
-; GFX10-NEXT: s_cselect_b32 s0, s2, s0
-; GFX10-NEXT: s_lshr_b32 s1, s2, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 9
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
-; GFX10-NEXT: s_lshr_b32 s1, s2, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 10
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
-; GFX10-NEXT: s_lshr_b32 s1, s2, 24
-; GFX10-NEXT: s_cmp_eq_u32 s4, 11
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 12
-; GFX10-NEXT: s_cselect_b32 s0, s3, s0
-; GFX10-NEXT: s_lshr_b32 s1, s3, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 13
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
-; GFX10-NEXT: s_lshr_b32 s1, s3, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 14
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
-; GFX10-NEXT: s_lshr_b32 s1, s3, 24
-; GFX10-NEXT: s_cmp_eq_u32 s4, 15
-; GFX10-NEXT: s_cselect_b32 s0, s1, s0
+; GFX10-NEXT: s_and_b32 s0, s4, 15
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_ashr_i32 s1, s0, 31
+; GFX10-NEXT: s_add_u32 s0, s2, s0
+; GFX10-NEXT: s_addc_u32 s1, s3, s1
+; GFX10-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s5, s0, 8
-; GFX11-NEXT: s_cmp_eq_u32 s4, 1
-; GFX11-NEXT: s_cselect_b32 s5, s5, s0
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 2
-; GFX11-NEXT: s_cselect_b32 s5, s6, s5
-; GFX11-NEXT: s_lshr_b32 s0, s0, 24
-; GFX11-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-NEXT: s_cselect_b32 s0, s0, s5
-; GFX11-NEXT: s_cmp_eq_u32 s4, 4
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-NEXT: s_cmp_eq_u32 s4, 5
-; GFX11-NEXT: s_cselect_b32 s0, s5, s0
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 6
-; GFX11-NEXT: s_cselect_b32 s0, s5, s0
-; GFX11-NEXT: s_lshr_b32 s1, s1, 24
-; GFX11-NEXT: s_cmp_eq_u32 s4, 7
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 8
-; GFX11-NEXT: s_cselect_b32 s0, s2, s0
-; GFX11-NEXT: s_lshr_b32 s1, s2, 8
-; GFX11-NEXT: s_cmp_eq_u32 s4, 9
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 10
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_lshr_b32 s1, s2, 24
-; GFX11-NEXT: s_cmp_eq_u32 s4, 11
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 12
-; GFX11-NEXT: s_cselect_b32 s0, s3, s0
-; GFX11-NEXT: s_lshr_b32 s1, s3, 8
-; GFX11-NEXT: s_cmp_eq_u32 s4, 13
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_lshr_b32 s1, s3, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 14
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_lshr_b32 s1, s3, 24
-; GFX11-NEXT: s_cmp_eq_u32 s4, 15
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
+; GFX11-NEXT: s_and_b32 s0, s4, 15
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_ashr_i32 s1, s0, 31
+; GFX11-NEXT: s_add_u32 s0, s2, s0
+; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: global_load_u8 v0, v0, s[0:1]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <16 x i8>, ptr addrspace(4) %ptr
%element = extractelement <16 x i8> %vector, i32 %idx
@@ -1681,326 +1767,65 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_sgpr_idx(ptr addrspace(4) inreg %
define amdgpu_ps i8 @extractelement_vgpr_v16i8_sgpr_idx(ptr addrspace(1) %ptr, i32 inreg %idx) {
; GFX9-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
-; GFX9-NEXT: s_cmp_eq_u32 s2, 1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 2
+; GFX9-NEXT: s_and_b32 s0, s2, 15
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 4
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 5
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 6
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 7
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 24, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 9
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 10
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 11
-; GFX9-NEXT: v_lshrrev_b32_e32 v12, 24, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 12
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v12, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 13
-; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 14
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v13, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 15
-; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GFX8-NEXT: s_cmp_eq_u32 s2, 1
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 2
+; GFX8-NEXT: s_and_b32 s0, s2, 15
+; GFX8-NEXT: s_ashr_i32 s1, s0, 31
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 4
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 6
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 7
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 9
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 8, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 10
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 11
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 24, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 12
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v12, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 13
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 14
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v13, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 15
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_cmp_eq_u32 s2, 1
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 2
+; GFX7-NEXT: s_and_b32 s0, s2, 15
+; GFX7-NEXT: s_ashr_i32 s1, s0, 31
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 3
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 4
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 5
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 6
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 7
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 24, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 9
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 8, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 10
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 11
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 24, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 12
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v12, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 13
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 14
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v13, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 15
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v14, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v15, vcc
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
-; GFX10-NEXT: s_cmp_eq_u32 s2, 1
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 2
+; GFX10-NEXT: s_and_b32 s0, s2, 15
+; GFX10-NEXT: s_ashr_i32 s1, s0, 31
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v0, v4, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 3
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 4
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX10-NEXT: s_cmp_eq_u32 s2, 5
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 6
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 7
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 8
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX10-NEXT: s_cmp_eq_u32 s2, 9
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 10
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v2
-; GFX10-NEXT: s_cmp_eq_u32 s2, 11
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 12
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX10-NEXT: s_cmp_eq_u32 s2, 13
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 14
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 15
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off
-; GFX11-NEXT: s_cmp_eq_u32 s2, 1
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 2
+; GFX11-NEXT: s_and_b32 s0, s2, 15
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_ashr_i32 s1, s0, 31
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 24, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b16 v4.l, v0.l, v4.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 3
-; GFX11-NEXT: v_cndmask_b16 v0.l, v4.l, v0.h, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 4
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v5.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 5
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 24, v1
-; GFX11-NEXT: s_cmp_eq_u32 s2, 7
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.h, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 8
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 9
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 24, v2
-; GFX11-NEXT: s_cmp_eq_u32 s2, 11
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v2.h, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 12
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-NEXT: s_cmp_eq_u32 s2, 13
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 24, v3
-; GFX11-NEXT: s_cmp_eq_u32 s2, 15
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v3.h, s0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <16 x i8>, ptr addrspace(1) %ptr
@@ -2012,248 +1837,60 @@ define i8 @extractelement_vgpr_v16i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 3, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 7, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 10, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 11, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[14:15], 14, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[16:17], 15, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, v0, s[18:19]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 9, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[10:11]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v5, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v5, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 12, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 13, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[14:15]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v6, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b64 vcc, s[16:17]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v6, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[0:1]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 3, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 7, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 10, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 11, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 14, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[16:17], 15, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 8, v3
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v0, s[18:19]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 9, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[10:11]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v5, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v5, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 12, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 13, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[14:15]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v6, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_mov_b64 vcc, s[16:17]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v6, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX7-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 8, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 24, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 24, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 8, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 24, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v12, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v13, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 8, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v14, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v16, 24, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v15, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 1, v2
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 8, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v0, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 3, v2
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 5, v2
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v3, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v1, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 7, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 9, v2
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v1, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 11, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v6
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 13, v2
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v1, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 15, v2
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 24, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v2
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v3.h, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 24, v4
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v4.h, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 8, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v5.l, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 10, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 24, v5
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v5.h, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 12, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v6
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v6.l, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 14, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 24, v6
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v6.h, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX11-NEXT: s_setpc_b64 s[30:31]
%vector = load <16 x i8>, ptr addrspace(1) %ptr
%element = extractelement <16 x i8> %vector, i32 %idx
@@ -2261,177 +1898,69 @@ define i8 @extractelement_vgpr_v16i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
}
define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
-; GCN-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshr_b32 s4, s0, 8
-; GCN-NEXT: v_mov_b32_e32 v1, s0
-; GCN-NEXT: v_mov_b32_e32 v2, s4
-; GCN-NEXT: s_lshr_b32 s4, s0, 16
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s4
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GCN-NEXT: s_lshr_b32 s0, s0, 24
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s1
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GCN-NEXT: s_lshr_b32 s0, s1, 8
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GCN-NEXT: s_lshr_b32 s0, s1, 16
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
-; GCN-NEXT: s_lshr_b32 s0, s1, 24
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s2
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 8, v0
-; GCN-NEXT: s_lshr_b32 s0, s2, 8
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 9, v0
-; GCN-NEXT: s_lshr_b32 s0, s2, 16
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 10, v0
-; GCN-NEXT: s_lshr_b32 s0, s2, 24
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 11, v0
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 12, v0
-; GCN-NEXT: s_lshr_b32 s0, s3, 8
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 13, v0
-; GCN-NEXT: s_lshr_b32 s0, s3, 16
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 14, v0
-; GCN-NEXT: s_lshr_b32 s0, s3, 24
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 15, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GCN-NEXT: v_readfirstlane_b32 s0, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_and_b32_e32 v2, 15, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX7-NEXT: s_mov_b32 s0, s2
+; GFX7-NEXT: s_mov_b32 s1, s3
+; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s4, s0, 8
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshr_b32 s4, s0, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT: s_lshr_b32 s0, s1, 8
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX10-NEXT: s_lshr_b32 s0, s1, 16
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT: s_lshr_b32 s0, s1, 24
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v0
-; GFX10-NEXT: s_lshr_b32 s0, s2, 8
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v0
-; GFX10-NEXT: s_lshr_b32 s0, s2, 16
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v0
-; GFX10-NEXT: s_lshr_b32 s0, s2, 24
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v0
-; GFX10-NEXT: s_lshr_b32 s0, s3, 8
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s3, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v0
-; GFX10-NEXT: s_lshr_b32 s0, s3, 16
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v0
-; GFX10-NEXT: s_lshr_b32 s0, s3, 24
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX10-NEXT: v_and_b32_e32 v2, 15, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[4:7], s[2:3], 0x0
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s0, s4, 8
-; GFX11-NEXT: s_lshr_b32 s1, s4, 16
-; GFX11-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v1.l, s4, v1.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, s0
-; GFX11-NEXT: s_lshr_b32 s1, s4, 24
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 4, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX11-NEXT: s_lshr_b32 s1, s5, 8
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s5, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, vcc_lo
-; GFX11-NEXT: s_lshr_b32 s1, s5, 16
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, s0
-; GFX11-NEXT: s_lshr_b32 s1, s5, 24
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 8, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v0
-; GFX11-NEXT: s_lshr_b32 s1, s6, 8
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s6, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 10, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, vcc_lo
-; GFX11-NEXT: s_lshr_b32 s1, s6, 16
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v0
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, s0
-; GFX11-NEXT: s_lshr_b32 s1, s6, 24
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 12, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v0
-; GFX11-NEXT: s_lshr_b32 s1, s7, 8
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s7, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 14, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, vcc_lo
-; GFX11-NEXT: s_lshr_b32 s1, s7, 16
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v0
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s1, s0
-; GFX11-NEXT: s_lshr_b32 s0, s7, 24
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b16 v0.l, v1.l, s0, vcc_lo
+; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 15, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: global_load_u8 v0, v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%vector = load <16 x i8>, ptr addrspace(4) %ptr
@@ -3156,3 +2685,5 @@ define i8 @extractelement_vgpr_v16i8_idx15(ptr addrspace(1) %ptr) {
%element = extractelement <16 x i8> %vector, i32 15
ret i8 %element
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
index 0a13398d5b875..9e11615d80ed1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GPRIDX %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,MOVREL %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GPRIDX %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,MOVREL %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
define float @dyn_extract_v8f32_const_s_v(i32 %sel) {
; GCN-LABEL: dyn_extract_v8f32_const_s_v:
@@ -583,34 +583,34 @@ define amdgpu_ps void @dyn_extract_v8i64_v_s(<8 x i64> %vec, i32 inreg %sel) {
; GPRIDX: ; %bb.0: ; %entry
; GPRIDX-NEXT: s_lshl_b32 s0, s2, 1
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
-; GPRIDX-NEXT: v_mov_b32_e32 v16, v1
-; GPRIDX-NEXT: v_mov_b32_e32 v15, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v16, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v17, v1
; GPRIDX-NEXT: s_set_gpr_idx_off
-; GPRIDX-NEXT: global_store_dwordx2 v[0:1], v[15:16], off
+; GPRIDX-NEXT: global_store_dwordx2 v[0:1], v[16:17], off
; GPRIDX-NEXT: s_endpgm
;
; MOVREL-LABEL: dyn_extract_v8i64_v_s:
; MOVREL: ; %bb.0: ; %entry
; MOVREL-NEXT: s_lshl_b32 m0, s2, 1
-; MOVREL-NEXT: v_movrels_b32_e32 v16, v1
-; MOVREL-NEXT: v_movrels_b32_e32 v15, v0
-; MOVREL-NEXT: flat_store_dwordx2 v[0:1], v[15:16]
+; MOVREL-NEXT: v_movrels_b32_e32 v16, v0
+; MOVREL-NEXT: v_movrels_b32_e32 v17, v1
+; MOVREL-NEXT: flat_store_dwordx2 v[0:1], v[16:17]
; MOVREL-NEXT: s_endpgm
;
; GFX10-LABEL: dyn_extract_v8i64_v_s:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_lshl_b32 m0, s2, 1
-; GFX10-NEXT: v_movrels_b32_e32 v16, v1
-; GFX10-NEXT: v_movrels_b32_e32 v15, v0
-; GFX10-NEXT: global_store_dwordx2 v[0:1], v[15:16], off
+; GFX10-NEXT: v_movrels_b32_e32 v16, v0
+; GFX10-NEXT: v_movrels_b32_e32 v17, v1
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[16:17], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: dyn_extract_v8i64_v_s:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_lshl_b32 m0, s2, 1
-; GFX11-NEXT: v_movrels_b32_e32 v16, v1
-; GFX11-NEXT: v_movrels_b32_e32 v15, v0
-; GFX11-NEXT: global_store_b64 v[0:1], v[15:16], off
+; GFX11-NEXT: v_movrels_b32_e32 v16, v0
+; GFX11-NEXT: v_movrels_b32_e32 v17, v1
+; GFX11-NEXT: global_store_b64 v[0:1], v[16:17], off
; GFX11-NEXT: s_endpgm
entry:
%ext = extractelement <8 x i64> %vec, i32 %sel
@@ -1703,20 +1703,20 @@ define amdgpu_ps double @dyn_extract_v16f64_v_s(<16 x double> %vec, i32 inreg %s
; GPRIDX: ; %bb.0: ; %entry
; GPRIDX-NEXT: s_lshl_b32 s0, s2, 1
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
-; GPRIDX-NEXT: v_mov_b32_e32 v32, v1
-; GPRIDX-NEXT: v_mov_b32_e32 v0, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v32, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v0, v1
; GPRIDX-NEXT: s_set_gpr_idx_off
-; GPRIDX-NEXT: v_readfirstlane_b32 s0, v0
-; GPRIDX-NEXT: v_readfirstlane_b32 s1, v32
+; GPRIDX-NEXT: v_readfirstlane_b32 s0, v32
+; GPRIDX-NEXT: v_readfirstlane_b32 s1, v0
; GPRIDX-NEXT: ; return to shader part epilog
;
; MOVREL-LABEL: dyn_extract_v16f64_v_s:
; MOVREL: ; %bb.0: ; %entry
; MOVREL-NEXT: s_lshl_b32 m0, s2, 1
-; MOVREL-NEXT: v_movrels_b32_e32 v32, v1
-; MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; MOVREL-NEXT: v_readfirstlane_b32 s0, v0
-; MOVREL-NEXT: v_readfirstlane_b32 s1, v32
+; MOVREL-NEXT: v_movrels_b32_e32 v32, v0
+; MOVREL-NEXT: v_movrels_b32_e32 v0, v1
+; MOVREL-NEXT: v_readfirstlane_b32 s0, v32
+; MOVREL-NEXT: v_readfirstlane_b32 s1, v0
; MOVREL-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: dyn_extract_v16f64_v_s:
@@ -2360,20 +2360,20 @@ define amdgpu_ps double @dyn_extract_v6f64_v_s(<6 x double> %vec, i32 inreg %sel
; GPRIDX: ; %bb.0: ; %entry
; GPRIDX-NEXT: s_lshl_b32 s0, s2, 1
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
-; GPRIDX-NEXT: v_mov_b32_e32 v12, v1
-; GPRIDX-NEXT: v_mov_b32_e32 v0, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v12, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v0, v1
; GPRIDX-NEXT: s_set_gpr_idx_off
-; GPRIDX-NEXT: v_readfirstlane_b32 s0, v0
-; GPRIDX-NEXT: v_readfirstlane_b32 s1, v12
+; GPRIDX-NEXT: v_readfirstlane_b32 s0, v12
+; GPRIDX-NEXT: v_readfirstlane_b32 s1, v0
; GPRIDX-NEXT: ; return to shader part epilog
;
; MOVREL-LABEL: dyn_extract_v6f64_v_s:
; MOVREL: ; %bb.0: ; %entry
; MOVREL-NEXT: s_lshl_b32 m0, s2, 1
-; MOVREL-NEXT: v_movrels_b32_e32 v12, v1
-; MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; MOVREL-NEXT: v_readfirstlane_b32 s0, v0
-; MOVREL-NEXT: v_readfirstlane_b32 s1, v12
+; MOVREL-NEXT: v_movrels_b32_e32 v12, v0
+; MOVREL-NEXT: v_movrels_b32_e32 v0, v1
+; MOVREL-NEXT: v_readfirstlane_b32 s0, v12
+; MOVREL-NEXT: v_readfirstlane_b32 s1, v0
; MOVREL-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: dyn_extract_v6f64_v_s:
@@ -2746,20 +2746,20 @@ define amdgpu_ps double @dyn_extract_v7f64_v_s(<7 x double> %vec, i32 inreg %sel
; GPRIDX: ; %bb.0: ; %entry
; GPRIDX-NEXT: s_lshl_b32 s0, s2, 1
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
-; GPRIDX-NEXT: v_mov_b32_e32 v14, v1
-; GPRIDX-NEXT: v_mov_b32_e32 v0, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v14, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v0, v1
; GPRIDX-NEXT: s_set_gpr_idx_off
-; GPRIDX-NEXT: v_readfirstlane_b32 s0, v0
-; GPRIDX-NEXT: v_readfirstlane_b32 s1, v14
+; GPRIDX-NEXT: v_readfirstlane_b32 s0, v14
+; GPRIDX-NEXT: v_readfirstlane_b32 s1, v0
; GPRIDX-NEXT: ; return to shader part epilog
;
; MOVREL-LABEL: dyn_extract_v7f64_v_s:
; MOVREL: ; %bb.0: ; %entry
; MOVREL-NEXT: s_lshl_b32 m0, s2, 1
-; MOVREL-NEXT: v_movrels_b32_e32 v14, v1
-; MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; MOVREL-NEXT: v_readfirstlane_b32 s0, v0
-; MOVREL-NEXT: v_readfirstlane_b32 s1, v14
+; MOVREL-NEXT: v_movrels_b32_e32 v14, v0
+; MOVREL-NEXT: v_movrels_b32_e32 v0, v1
+; MOVREL-NEXT: v_readfirstlane_b32 s0, v14
+; MOVREL-NEXT: v_readfirstlane_b32 s1, v0
; MOVREL-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: dyn_extract_v7f64_v_s:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
index 2a46b4a150ff2..f5032ef159410 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
@@ -174,8 +174,11 @@ define amdgpu_ps void @s_fabs_v2f16(<2 x half> inreg %in, ptr addrspace(1) %out)
;
; GFX12-LABEL: s_fabs_v2f16:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff7fff
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_lshr_b32 s1, s0, 16
+; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX12-NEXT: s_and_b32 s1, s1, 0x7fff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
@@ -197,10 +200,13 @@ define amdgpu_ps void @s_fabs_v2f16_salu_use(<2 x half> inreg %in, i32 inreg %va
;
; GFX12-LABEL: s_fabs_v2f16_salu_use:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff7fff
+; GFX12-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX12-NEXT: s_and_b32 s2, s2, 0x7fff
; GFX12-NEXT: s_cmp_eq_u32 s1, 0
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cselect_b32 s0, s0, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
index 550ef55ab570d..d9678cb7aae37 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
@@ -109,10 +109,21 @@ define amdgpu_ps void @fadd_s64_div(double %a, double %b, ptr addrspace(1) %ptr)
}
define amdgpu_ps <2 x half> @fadd_v2s16_uniform(<2 x half> inreg %a, <2 x half> inreg %b) {
-; GCN-LABEL: fadd_v2s16_uniform:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_pk_add_f16 v0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX11-LABEL: fadd_v2s16_uniform:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_pk_add_f16 v0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v2s16_uniform:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-NEXT: s_lshr_b32 s3, s1, 16
+; GFX12-NEXT: s_add_f16 s0, s0, s1
+; GFX12-NEXT: s_add_f16 s1, s2, s3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
%fadd = fadd <2 x half> %a, %b
ret <2 x half> %fadd
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
index e8e833e8cc47d..47cb6bdfde09c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
@@ -2104,55 +2104,11 @@ define amdgpu_ps float @dont_crash_after_fma_mix_select_attempt(float inreg %x,
define amdgpu_ps half @fma_s16_uniform(half inreg %a, half inreg %b, half inreg %c) {
; GFX6-LABEL: fma_s16_uniform:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, s1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, s0
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
-; GFX6-NEXT: v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
-; GFX6-NEXT: v_readfirstlane_b32 s0, v1
-; GFX6-NEXT: v_readfirstlane_b32 s1, v0
-; GFX6-NEXT: s_and_b32 s2, s0, 0x1ff
-; GFX6-NEXT: s_or_b32 s1, s2, s1
-; GFX6-NEXT: s_cselect_b32 s1, 1, 0
-; GFX6-NEXT: s_lshr_b32 s2, s0, 8
-; GFX6-NEXT: s_bfe_u32 s3, s0, 0xb0014
-; GFX6-NEXT: s_and_b32 s2, s2, 0xffe
-; GFX6-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX6-NEXT: s_or_b32 s1, s2, s1
-; GFX6-NEXT: v_med3_i32 v0, s4, 0, 13
-; GFX6-NEXT: s_or_b32 s2, s1, 0x1000
-; GFX6-NEXT: v_readfirstlane_b32 s4, v0
-; GFX6-NEXT: s_lshr_b32 s5, s2, s4
-; GFX6-NEXT: s_lshl_b32 s4, s5, s4
-; GFX6-NEXT: s_cmp_lg_u32 s4, s2
-; GFX6-NEXT: s_cselect_b32 s2, 1, 0
-; GFX6-NEXT: s_addk_i32 s3, 0xfc10
-; GFX6-NEXT: s_lshl_b32 s4, s3, 12
-; GFX6-NEXT: s_or_b32 s2, s5, s2
-; GFX6-NEXT: s_or_b32 s4, s1, s4
-; GFX6-NEXT: s_cmp_lt_i32 s3, 1
-; GFX6-NEXT: s_cselect_b32 s2, s2, s4
-; GFX6-NEXT: s_and_b32 s4, s2, 7
-; GFX6-NEXT: s_cmp_gt_i32 s4, 5
-; GFX6-NEXT: s_cselect_b32 s5, 1, 0
-; GFX6-NEXT: s_cmp_eq_u32 s4, 3
-; GFX6-NEXT: s_cselect_b32 s4, 1, 0
-; GFX6-NEXT: s_or_b32 s4, s4, s5
-; GFX6-NEXT: s_lshr_b32 s2, s2, 2
-; GFX6-NEXT: s_add_i32 s2, s2, s4
-; GFX6-NEXT: s_cmp_lt_i32 s3, 31
-; GFX6-NEXT: s_cselect_b32 s2, s2, 0x7c00
-; GFX6-NEXT: s_cmp_lg_u32 s1, 0
-; GFX6-NEXT: s_movk_i32 s1, 0x7e00
-; GFX6-NEXT: s_cselect_b32 s1, s1, 0x7c00
-; GFX6-NEXT: s_cmpk_eq_i32 s3, 0x40f
-; GFX6-NEXT: s_cselect_b32 s1, s1, s2
-; GFX6-NEXT: s_lshr_b32 s0, s0, 16
-; GFX6-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX6-NEXT: s_or_b32 s0, s0, s1
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, s1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, s2
+; GFX6-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: fma_s16_uniform:
@@ -2471,8 +2427,13 @@ define amdgpu_ps <2 x half> @fma_v2s16_uniform(<2 x half> inreg %a, <2 x half> i
;
; GFX12-LABEL: fma_v2s16_uniform:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: v_pk_fma_f16 v0, s0, s1, v0
+; GFX12-NEXT: s_lshr_b32 s3, s0, 16
+; GFX12-NEXT: s_lshr_b32 s4, s1, 16
+; GFX12-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-NEXT: s_fmac_f16 s2, s0, s1
+; GFX12-NEXT: s_fmac_f16 s5, s3, s4
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s5
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
%fma = call <2 x half> @llvm.fma.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)
ret <2 x half> %fma
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
index 447127050f8a0..207ae37d814fe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
@@ -109,10 +109,21 @@ define amdgpu_ps void @fmul_s64_div(double %a, double %b, ptr addrspace(1) %ptr)
}
define amdgpu_ps <2 x half> @fmul_v2s16_uniform(<2 x half> inreg %a, <2 x half> inreg %b) {
-; GCN-LABEL: fmul_v2s16_uniform:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_pk_mul_f16 v0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX11-LABEL: fmul_v2s16_uniform:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_pk_mul_f16 v0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v2s16_uniform:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-NEXT: s_lshr_b32 s3, s1, 16
+; GFX12-NEXT: s_mul_f16 s0, s0, s1
+; GFX12-NEXT: s_mul_f16 s1, s2, s3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
%result = fmul <2 x half> %a, %b
ret <2 x half> %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
index f41ff7ba31a5b..71e2d7e11e4e3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
@@ -174,8 +174,11 @@ define amdgpu_ps void @s_fneg_v2f16(<2 x half> inreg %in, ptr addrspace(1) %out)
;
; GFX12-LABEL: s_fneg_v2f16:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_lshr_b32 s1, s0, 16
+; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX12-NEXT: s_xor_b32 s1, s1, 0x8000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
@@ -197,10 +200,13 @@ define amdgpu_ps void @s_fneg_v2f16_salu_use(<2 x half> inreg %in, i32 inreg %va
;
; GFX12-LABEL: s_fneg_v2f16_salu_use:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_xor_b32 s0, s0, 0x80008000
+; GFX12-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX12-NEXT: s_xor_b32 s2, s2, 0x8000
; GFX12-NEXT: s_cmp_eq_u32 s1, 0
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cselect_b32 s0, s0, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index 35e0c0f06e8e4..b330c77851493 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -597,16 +597,26 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
define amdgpu_ps <2 x half> @fptrunc_v2f32_to_v2f16_uniform(<2 x float> inreg %a) {
; GFX11-FAKE16-LABEL: fptrunc_v2f32_to_v2f16_uniform:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s1
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s0
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: fptrunc_v2f32_to_v2f16_uniform:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, s1
; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: fptrunc_v2f32_to_v2f16_uniform:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
index a17347d4b0c61..18eaf0df72ed1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
@@ -218,90 +218,41 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1, ptr addrspace(1) %in2) #0 {
; CI-LABEL: fast_frem_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: s_mov_b32 s6, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s0, s8
-; CI-NEXT: s_mov_b32 s1, s9
-; CI-NEXT: s_mov_b32 s8, s10
-; CI-NEXT: s_mov_b32 s9, s11
-; CI-NEXT: s_mov_b32 s10, s2
-; CI-NEXT: s_mov_b32 s11, s3
-; CI-NEXT: s_mov_b32 s7, s3
-; CI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
-; CI-NEXT: buffer_load_ushort v1, off, s[4:7], 0 offset:8
-; CI-NEXT: s_waitcnt vmcnt(1)
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; CI-NEXT: v_div_scale_f32 v1, s[4:5], v4, v4, v2
-; CI-NEXT: v_div_scale_f32 v0, vcc, v2, v4, v2
-; CI-NEXT: v_rcp_f32_e32 v3, v1
+; CI-NEXT: s_load_dword s6, s[2:3], 0x0
+; CI-NEXT: s_load_dword s4, s[4:5], 0x2
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s6
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; CI-NEXT: v_div_scale_f32 v2, s[2:3], v1, v1, v0
+; CI-NEXT: v_div_scale_f32 v3, s[2:3], v0, v1, v0
+; CI-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
+; CI-NEXT: v_rcp_f32_e32 v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v5, -v1, v3, 1.0
-; CI-NEXT: v_fma_f32 v3, v5, v3, v3
-; CI-NEXT: v_mul_f32_e32 v5, v0, v3
-; CI-NEXT: v_fma_f32 v6, -v1, v5, v0
-; CI-NEXT: v_fma_f32 v5, v6, v3, v5
-; CI-NEXT: v_fma_f32 v0, -v1, v5, v0
+; CI-NEXT: v_fma_f32 v5, -v2, v4, 1.0
+; CI-NEXT: v_fma_f32 v4, v5, v4, v4
+; CI-NEXT: v_mul_f32_e32 v5, v3, v4
+; CI-NEXT: v_fma_f32 v6, -v2, v5, v3
+; CI-NEXT: v_fma_f32 v5, v6, v4, v5
+; CI-NEXT: v_fma_f32 v2, -v2, v5, v3
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v0, v0, v3, v5
-; CI-NEXT: v_div_fixup_f32 v0, v0, v4, v2
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: v_div_fixup_f32 v0, v2, v1, v0
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
-; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; CI-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
+; CI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; CI-NEXT: v_cvt_f32_f16_e32 v2, s6
; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; CI-NEXT: v_trunc_f32_e32 v0, v0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; CI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; CI-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], v[2:3]
-; CI-NEXT: v_readfirstlane_b32 s4, v1
-; CI-NEXT: s_and_b32 s5, s4, 0x1ff
-; CI-NEXT: v_readfirstlane_b32 s6, v0
-; CI-NEXT: s_or_b32 s5, s5, s6
-; CI-NEXT: s_cselect_b32 s5, 1, 0
-; CI-NEXT: s_lshr_b32 s6, s4, 8
-; CI-NEXT: s_bfe_u32 s7, s4, 0xb0014
-; CI-NEXT: s_and_b32 s6, s6, 0xffe
-; CI-NEXT: s_sub_i32 s8, 0x3f1, s7
-; CI-NEXT: s_or_b32 s5, s6, s5
-; CI-NEXT: v_med3_i32 v0, s8, 0, 13
-; CI-NEXT: s_or_b32 s6, s5, 0x1000
-; CI-NEXT: v_readfirstlane_b32 s8, v0
-; CI-NEXT: s_lshr_b32 s9, s6, s8
-; CI-NEXT: s_lshl_b32 s8, s9, s8
-; CI-NEXT: s_cmp_lg_u32 s8, s6
-; CI-NEXT: s_cselect_b32 s6, 1, 0
-; CI-NEXT: s_addk_i32 s7, 0xfc10
-; CI-NEXT: s_lshl_b32 s8, s7, 12
-; CI-NEXT: s_or_b32 s6, s9, s6
-; CI-NEXT: s_or_b32 s8, s5, s8
-; CI-NEXT: s_cmp_lt_i32 s7, 1
-; CI-NEXT: s_cselect_b32 s6, s6, s8
-; CI-NEXT: s_and_b32 s8, s6, 7
-; CI-NEXT: s_cmp_gt_i32 s8, 5
-; CI-NEXT: s_cselect_b32 s9, 1, 0
-; CI-NEXT: s_cmp_eq_u32 s8, 3
-; CI-NEXT: s_cselect_b32 s8, 1, 0
-; CI-NEXT: s_or_b32 s8, s8, s9
-; CI-NEXT: s_lshr_b32 s6, s6, 2
-; CI-NEXT: s_add_i32 s6, s6, s8
-; CI-NEXT: s_cmp_lt_i32 s7, 31
-; CI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; CI-NEXT: s_cmp_lg_u32 s5, 0
-; CI-NEXT: s_movk_i32 s5, 0x7e00
-; CI-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; CI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; CI-NEXT: s_cselect_b32 s5, s5, s6
-; CI-NEXT: s_lshr_b32 s4, s4, 16
-; CI-NEXT: s_and_b32 s4, s4, 0x8000
-; CI-NEXT: s_or_b32 s4, s4, s5
-; CI-NEXT: v_mov_b32_e32 v0, s4
+; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; CI-NEXT: v_fma_f32 v0, v0, v1, v2
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
;
@@ -310,32 +261,27 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-NEXT: s_load_dword s3, s[4:5], 0x8
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; VI-NEXT: v_cvt_f32_f16_e32 v2, s3
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_rcp_f32_e32 v3, v2
+; VI-NEXT: v_mul_f32_e32 v4, v0, v3
+; VI-NEXT: v_mad_f32 v5, -v2, v4, v0
+; VI-NEXT: v_mac_f32_e32 v4, v5, v3
+; VI-NEXT: v_mad_f32 v0, -v2, v4, v0
+; VI-NEXT: v_mul_f32_e32 v0, v0, v3
+; VI-NEXT: v_readfirstlane_b32 s3, v0
+; VI-NEXT: s_and_b32 s3, s3, 0xff800000
+; VI-NEXT: v_add_f32_e32 v0, s3, v4
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: v_div_fixup_f16 v0, v0, v1, s2
+; VI-NEXT: v_trunc_f16_e32 v0, v0
+; VI-NEXT: v_fma_f16 v2, -v0, v1, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_add_u32 s0, s4, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: s_addc_u32 s1, s5, 0
-; VI-NEXT: flat_load_ushort v4, v[2:3]
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_load_ushort v2, v[2:3]
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v2
-; VI-NEXT: v_rcp_f32_e32 v6, v5
-; VI-NEXT: v_mul_f32_e32 v7, v3, v6
-; VI-NEXT: v_mad_f32 v8, -v5, v7, v3
-; VI-NEXT: v_mac_f32_e32 v7, v8, v6
-; VI-NEXT: v_mad_f32 v3, -v5, v7, v3
-; VI-NEXT: v_mul_f32_e32 v3, v3, v6
-; VI-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; VI-NEXT: v_add_f32_e32 v3, v3, v7
-; VI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; VI-NEXT: v_div_fixup_f16 v3, v3, v2, v4
-; VI-NEXT: v_trunc_f16_e32 v3, v3
-; VI-NEXT: v_fma_f16 v2, -v3, v2, v4
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
%gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
@@ -349,90 +295,41 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1, ptr addrspace(1) %in2) #1 {
; CI-LABEL: unsafe_frem_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: s_mov_b32 s6, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s0, s8
-; CI-NEXT: s_mov_b32 s1, s9
-; CI-NEXT: s_mov_b32 s8, s10
-; CI-NEXT: s_mov_b32 s9, s11
-; CI-NEXT: s_mov_b32 s10, s2
-; CI-NEXT: s_mov_b32 s11, s3
-; CI-NEXT: s_mov_b32 s7, s3
-; CI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
-; CI-NEXT: buffer_load_ushort v1, off, s[4:7], 0 offset:8
-; CI-NEXT: s_waitcnt vmcnt(1)
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; CI-NEXT: v_div_scale_f32 v1, s[4:5], v4, v4, v2
-; CI-NEXT: v_div_scale_f32 v0, vcc, v2, v4, v2
-; CI-NEXT: v_rcp_f32_e32 v3, v1
+; CI-NEXT: s_load_dword s6, s[2:3], 0x0
+; CI-NEXT: s_load_dword s4, s[4:5], 0x2
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s6
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; CI-NEXT: v_div_scale_f32 v2, s[2:3], v1, v1, v0
+; CI-NEXT: v_div_scale_f32 v3, s[2:3], v0, v1, v0
+; CI-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
+; CI-NEXT: v_rcp_f32_e32 v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v5, -v1, v3, 1.0
-; CI-NEXT: v_fma_f32 v3, v5, v3, v3
-; CI-NEXT: v_mul_f32_e32 v5, v0, v3
-; CI-NEXT: v_fma_f32 v6, -v1, v5, v0
-; CI-NEXT: v_fma_f32 v5, v6, v3, v5
-; CI-NEXT: v_fma_f32 v0, -v1, v5, v0
+; CI-NEXT: v_fma_f32 v5, -v2, v4, 1.0
+; CI-NEXT: v_fma_f32 v4, v5, v4, v4
+; CI-NEXT: v_mul_f32_e32 v5, v3, v4
+; CI-NEXT: v_fma_f32 v6, -v2, v5, v3
+; CI-NEXT: v_fma_f32 v5, v6, v4, v5
+; CI-NEXT: v_fma_f32 v2, -v2, v5, v3
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v0, v0, v3, v5
-; CI-NEXT: v_div_fixup_f32 v0, v0, v4, v2
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: v_div_fixup_f32 v0, v2, v1, v0
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
-; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; CI-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
+; CI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; CI-NEXT: v_cvt_f32_f16_e32 v2, s6
; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; CI-NEXT: v_trunc_f32_e32 v0, v0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; CI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; CI-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], v[2:3]
-; CI-NEXT: v_readfirstlane_b32 s4, v1
-; CI-NEXT: s_and_b32 s5, s4, 0x1ff
-; CI-NEXT: v_readfirstlane_b32 s6, v0
-; CI-NEXT: s_or_b32 s5, s5, s6
-; CI-NEXT: s_cselect_b32 s5, 1, 0
-; CI-NEXT: s_lshr_b32 s6, s4, 8
-; CI-NEXT: s_bfe_u32 s7, s4, 0xb0014
-; CI-NEXT: s_and_b32 s6, s6, 0xffe
-; CI-NEXT: s_sub_i32 s8, 0x3f1, s7
-; CI-NEXT: s_or_b32 s5, s6, s5
-; CI-NEXT: v_med3_i32 v0, s8, 0, 13
-; CI-NEXT: s_or_b32 s6, s5, 0x1000
-; CI-NEXT: v_readfirstlane_b32 s8, v0
-; CI-NEXT: s_lshr_b32 s9, s6, s8
-; CI-NEXT: s_lshl_b32 s8, s9, s8
-; CI-NEXT: s_cmp_lg_u32 s8, s6
-; CI-NEXT: s_cselect_b32 s6, 1, 0
-; CI-NEXT: s_addk_i32 s7, 0xfc10
-; CI-NEXT: s_lshl_b32 s8, s7, 12
-; CI-NEXT: s_or_b32 s6, s9, s6
-; CI-NEXT: s_or_b32 s8, s5, s8
-; CI-NEXT: s_cmp_lt_i32 s7, 1
-; CI-NEXT: s_cselect_b32 s6, s6, s8
-; CI-NEXT: s_and_b32 s8, s6, 7
-; CI-NEXT: s_cmp_gt_i32 s8, 5
-; CI-NEXT: s_cselect_b32 s9, 1, 0
-; CI-NEXT: s_cmp_eq_u32 s8, 3
-; CI-NEXT: s_cselect_b32 s8, 1, 0
-; CI-NEXT: s_or_b32 s8, s8, s9
-; CI-NEXT: s_lshr_b32 s6, s6, 2
-; CI-NEXT: s_add_i32 s6, s6, s8
-; CI-NEXT: s_cmp_lt_i32 s7, 31
-; CI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; CI-NEXT: s_cmp_lg_u32 s5, 0
-; CI-NEXT: s_movk_i32 s5, 0x7e00
-; CI-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; CI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; CI-NEXT: s_cselect_b32 s5, s5, s6
-; CI-NEXT: s_lshr_b32 s4, s4, 16
-; CI-NEXT: s_and_b32 s4, s4, 0x8000
-; CI-NEXT: s_or_b32 s4, s4, s5
-; CI-NEXT: v_mov_b32_e32 v0, s4
+; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; CI-NEXT: v_fma_f32 v0, v0, v1, v2
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: buffer_store_short v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
;
@@ -441,32 +338,27 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-NEXT: s_load_dword s3, s[4:5], 0x8
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; VI-NEXT: v_cvt_f32_f16_e32 v2, s3
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_rcp_f32_e32 v3, v2
+; VI-NEXT: v_mul_f32_e32 v4, v0, v3
+; VI-NEXT: v_mad_f32 v5, -v2, v4, v0
+; VI-NEXT: v_mac_f32_e32 v4, v5, v3
+; VI-NEXT: v_mad_f32 v0, -v2, v4, v0
+; VI-NEXT: v_mul_f32_e32 v0, v0, v3
+; VI-NEXT: v_readfirstlane_b32 s3, v0
+; VI-NEXT: s_and_b32 s3, s3, 0xff800000
+; VI-NEXT: v_add_f32_e32 v0, s3, v4
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: v_div_fixup_f16 v0, v0, v1, s2
+; VI-NEXT: v_trunc_f16_e32 v0, v0
+; VI-NEXT: v_fma_f16 v2, -v0, v1, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_add_u32 s0, s4, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: s_addc_u32 s1, s5, 0
-; VI-NEXT: flat_load_ushort v4, v[2:3]
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_load_ushort v2, v[2:3]
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v4
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v2
-; VI-NEXT: v_rcp_f32_e32 v6, v5
-; VI-NEXT: v_mul_f32_e32 v7, v3, v6
-; VI-NEXT: v_mad_f32 v8, -v5, v7, v3
-; VI-NEXT: v_mac_f32_e32 v7, v8, v6
-; VI-NEXT: v_mad_f32 v3, -v5, v7, v3
-; VI-NEXT: v_mul_f32_e32 v3, v3, v6
-; VI-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; VI-NEXT: v_add_f32_e32 v3, v3, v7
-; VI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; VI-NEXT: v_div_fixup_f16 v3, v3, v2, v4
-; VI-NEXT: v_trunc_f16_e32 v3, v3
-; VI-NEXT: v_fma_f16 v2, -v3, v2, v4
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
%gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 259fc1a97315e..f80a0ff197386 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -43,60 +43,152 @@ define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
;
; GFX8-LABEL: s_fshl_i7:
; GFX8: ; %bb.0:
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX8-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mul_i32 s4, s3, -7
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_add_i32 s3, s3, s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mul_i32 s3, s3, -7
+; GFX8-NEXT: s_add_i32 s2, s2, s3
+; GFX8-NEXT: s_cmp_ge_u32 s2, 7
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_add_i32 s4, s2, -7
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_cmp_ge_u32 s2, 7
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_add_i32 s4, s2, -7
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
; GFX8-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX8-NEXT: s_lshl_b32 s0, s0, 7
+; GFX8-NEXT: s_sub_i32 s3, 6, s2
+; GFX8-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s2
+; GFX8-NEXT: s_lshr_b32 s1, s1, 1
+; GFX8-NEXT: s_and_b32 s2, s3, 0x7f
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_mul_i32 s1, s2, 0x2493
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_mul_i32 s1, s1, 7
-; GFX8-NEXT: s_sub_i32 s1, s2, s1
-; GFX8-NEXT: s_lshl_b32 s0, s0, s1
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x90007
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshl_i7:
; GFX9: ; %bb.0:
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-NEXT: s_mul_i32 s4, s3, -7
+; GFX9-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX9-NEXT: s_add_i32 s3, s3, s4
+; GFX9-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX9-NEXT: s_mul_i32 s3, s3, -7
+; GFX9-NEXT: s_add_i32 s2, s2, s3
+; GFX9-NEXT: s_cmp_ge_u32 s2, 7
+; GFX9-NEXT: s_cselect_b32 s3, 1, 0
+; GFX9-NEXT: s_add_i32 s4, s2, -7
+; GFX9-NEXT: s_cmp_lg_u32 s3, 0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s2
+; GFX9-NEXT: s_cmp_ge_u32 s2, 7
+; GFX9-NEXT: s_cselect_b32 s3, 1, 0
+; GFX9-NEXT: s_add_i32 s4, s2, -7
+; GFX9-NEXT: s_cmp_lg_u32 s3, 0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s2
; GFX9-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX9-NEXT: s_lshl_b32 s0, s0, 7
+; GFX9-NEXT: s_sub_i32 s3, 6, s2
+; GFX9-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s1, s1, 1
+; GFX9-NEXT: s_and_b32 s2, s3, 0x7f
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
-; GFX9-NEXT: s_mul_i32 s1, s2, 0x2493
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: s_mul_i32 s1, s1, 7
-; GFX9-NEXT: s_sub_i32 s1, s2, s1
-; GFX9-NEXT: s_lshl_b32 s0, s0, s1
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x90007
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshl_i7:
; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
; GFX10-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-NEXT: s_mul_i32 s4, s3, -7
+; GFX10-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX10-NEXT: s_add_i32 s3, s3, s4
+; GFX10-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX10-NEXT: s_mul_i32 s3, s3, -7
+; GFX10-NEXT: s_add_i32 s2, s2, s3
+; GFX10-NEXT: s_cmp_ge_u32 s2, 7
+; GFX10-NEXT: s_cselect_b32 s3, 1, 0
+; GFX10-NEXT: s_add_i32 s4, s2, -7
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10-NEXT: s_cselect_b32 s2, s4, s2
+; GFX10-NEXT: s_cmp_ge_u32 s2, 7
+; GFX10-NEXT: s_cselect_b32 s3, 1, 0
+; GFX10-NEXT: s_add_i32 s4, s2, -7
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10-NEXT: s_cselect_b32 s2, s4, s2
; GFX10-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX10-NEXT: s_mul_i32 s3, s2, 0x2493
-; GFX10-NEXT: s_lshl_b32 s0, s0, 7
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
+; GFX10-NEXT: s_sub_i32 s3, 6, s2
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX10-NEXT: s_lshr_b32 s1, s1, 1
+; GFX10-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX10-NEXT: s_lshl_b32 s0, s0, s2
+; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
-; GFX10-NEXT: s_mul_i32 s3, s3, 7
-; GFX10-NEXT: s_sub_i32 s1, s2, s3
-; GFX10-NEXT: s_lshl_b32 s0, s0, s1
-; GFX10-NEXT: s_bfe_u32 s0, s0, 0x90007
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i7:
; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX11-NEXT: s_mul_i32 s3, s2, 0x2493
-; GFX11-NEXT: s_lshl_b32 s0, s0, 7
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_mul_i32 s3, s3, 7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, -7
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s1, s2, s3
-; GFX11-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-NEXT: s_mul_i32 s3, s3, -7
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
+; GFX11-NEXT: s_sub_i32 s3, 6, s2
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_bfe_u32 s0, s0, 0x90007
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
@@ -237,47 +329,55 @@ define amdgpu_ps i8 @s_fshl_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
;
; GFX8-LABEL: s_fshl_i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX8-NEXT: v_perm_b32 v0, s1, v0, v1
-; GFX8-NEXT: s_and_b32 s0, s2, 7
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX8-NEXT: v_bfe_u32 v0, v0, 8, 8
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s3, s2, 7
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_xor_b32 s2, s2, -1
+; GFX8-NEXT: s_lshr_b32 s1, s1, 1
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_lshl_b32 s0, s0, s3
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshl_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s1, v0, v1
-; GFX9-NEXT: s_and_b32 s0, s2, 7
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX9-NEXT: v_bfe_u32 v0, v0, 8, 8
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s3, s2, 7
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_xor_b32 s2, s2, -1
+; GFX9-NEXT: s_lshr_b32 s1, s1, 1
+; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_lshl_b32 s0, s0, s3
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshl_i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX10-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX10-NEXT: s_and_b32 s0, s2, 7
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_bfe_u32 v0, v0, 8, 8
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_and_b32 s3, s2, 7
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_xor_b32 s2, s2, -1
+; GFX10-NEXT: s_lshr_b32 s1, s1, 1
+; GFX10-NEXT: s_and_b32 s2, s2, 7
+; GFX10-NEXT: s_lshl_b32 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i8:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX11-NEXT: s_and_b32 s0, s2, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_bfe_u32 v0, v0, 8, 8
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_and_b32 s3, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_xor_b32 s2, s2, -1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
@@ -355,18 +455,47 @@ define i8 @v_fshl_i8(i8 %lhs, i8 %rhs, i8 %amt) {
}
define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
-; GCN-LABEL: s_fshl_i8_4:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b32 s0, s0, 4
-; GCN-NEXT: s_bfe_u32 s1, s1, 0x40004
-; GCN-NEXT: s_or_b32 s0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i8_4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b32 s0, s0, 4
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x40004
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i8_4:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshr_b32 s1, s1, 4
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i8_4:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, 4
+; GFX9-NEXT: s_lshr_b32 s1, s1, 4
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i8_4:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshr_b32 s1, s1, 4
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i8_4:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x40004
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 4)
@@ -433,18 +562,47 @@ define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
}
define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
-; GCN-LABEL: s_fshl_i8_5:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b32 s0, s0, 5
-; GCN-NEXT: s_bfe_u32 s1, s1, 0x50003
-; GCN-NEXT: s_or_b32 s0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i8_5:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b32 s0, s0, 5
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x50003
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i8_5:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 5
+; GFX8-NEXT: s_lshr_b32 s1, s1, 3
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i8_5:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, 5
+; GFX9-NEXT: s_lshr_b32 s1, s1, 3
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i8_5:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s0, s0, 5
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshr_b32 s1, s1, 3
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i8_5:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x50003
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 3
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 5)
@@ -536,65 +694,119 @@ define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
;
; GFX8-LABEL: s_fshl_v2i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0xc0c0105
-; GFX8-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX8-NEXT: v_perm_b32 v0, s1, v0, v2
-; GFX8-NEXT: s_and_b32 s0, s2, 7
-; GFX8-NEXT: s_mov_b32 s3, 0xc0c0105
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX8-NEXT: v_perm_b32 v0, v0, v1, s3
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_lshr_b32 s4, s1, 8
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_lshr_b32 s5, s2, 8
+; GFX8-NEXT: s_and_b32 s6, s2, 7
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_xor_b32 s2, s2, -1
+; GFX8-NEXT: s_lshr_b32 s1, s1, 1
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_lshr_b32 s3, s0, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, s6
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s5, 7
+; GFX8-NEXT: s_and_b32 s2, s4, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s3, s1
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_xor_b32 s3, s5, -1
+; GFX8-NEXT: s_lshr_b32 s2, s2, 1
+; GFX8-NEXT: s_and_b32 s3, s3, 7
+; GFX8-NEXT: s_lshr_b32 s2, s2, s3
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshl_v2i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX9-NEXT: v_perm_b32 v0, s1, v0, v2
-; GFX9-NEXT: s_and_b32 s0, s2, 7
-; GFX9-NEXT: s_mov_b32 s3, 0xc0c0105
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s3
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s4, s1, 8
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_lshr_b32 s5, s2, 8
+; GFX9-NEXT: s_and_b32 s6, s2, 7
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_xor_b32 s2, s2, -1
+; GFX9-NEXT: s_lshr_b32 s1, s1, 1
+; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_lshr_b32 s3, s0, 8
+; GFX9-NEXT: s_lshl_b32 s0, s0, s6
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: s_and_b32 s1, s5, 7
+; GFX9-NEXT: s_and_b32 s2, s4, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s3, s1
+; GFX9-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX9-NEXT: s_xor_b32 s3, s5, -1
+; GFX9-NEXT: s_lshr_b32 s2, s2, 1
+; GFX9-NEXT: s_and_b32 s3, s3, 7
+; GFX9-NEXT: s_lshr_b32 s2, s2, s3
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshl_v2i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX10-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX10-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX10-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX10-NEXT: s_and_b32 s1, s2, 7
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, s1, v1
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s4, s1, 8
+; GFX10-NEXT: s_lshr_b32 s5, s2, 8
+; GFX10-NEXT: s_and_b32 s6, s2, 7
+; GFX10-NEXT: s_and_b32 s4, s4, 0xff
+; GFX10-NEXT: s_lshr_b32 s3, s0, 8
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s0, s0, s6
+; GFX10-NEXT: s_and_b32 s6, s5, 7
+; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX10-NEXT: s_xor_b32 s5, s5, -1
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_xor_b32 s2, s2, -1
+; GFX10-NEXT: s_lshr_b32 s4, s4, 1
+; GFX10-NEXT: s_and_b32 s5, s5, 7
+; GFX10-NEXT: s_lshr_b32 s1, s1, 1
+; GFX10-NEXT: s_and_b32 s2, s2, 7
+; GFX10-NEXT: s_lshl_b32 s3, s3, s6
+; GFX10-NEXT: s_lshr_b32 s4, s4, s5
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
+; GFX10-NEXT: s_or_b32 s2, s3, s4
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: s_and_b32 s1, s2, 0xff
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v2i8:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX11-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX11-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX11-NEXT: s_and_b32 s1, s2, 7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, s1, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-NEXT: s_and_b32 s6, s2, 7
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_and_b32 s6, s5, 7
+; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-NEXT: s_xor_b32 s5, s5, -1
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_xor_b32 s2, s2, -1
+; GFX11-NEXT: s_lshr_b32 s4, s4, 1
+; GFX11-NEXT: s_and_b32 s5, s5, 7
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_lshl_b32 s3, s3, s6
+; GFX11-NEXT: s_lshr_b32 s4, s4, s5
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_or_b32 s2, s3, s4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_and_b32 s1, s2, 0xff
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -2921,44 +3133,44 @@ define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX8-LABEL: s_fshl_i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX8-NEXT: s_andn2_b32 s3, 15, s2
+; GFX8-NEXT: s_and_b32 s3, s2, 15
+; GFX8-NEXT: s_andn2_b32 s2, 15, s2
; GFX8-NEXT: s_lshr_b32 s1, s1, 1
-; GFX8-NEXT: s_and_b32 s2, s2, 15
-; GFX8-NEXT: s_lshr_b32 s1, s1, s3
-; GFX8-NEXT: s_lshl_b32 s0, s0, s2
+; GFX8-NEXT: s_lshl_b32 s0, s0, s3
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshl_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_andn2_b32 s3, 15, s2
+; GFX9-NEXT: s_and_b32 s3, s2, 15
+; GFX9-NEXT: s_andn2_b32 s2, 15, s2
; GFX9-NEXT: s_lshr_b32 s1, s1, 1
-; GFX9-NEXT: s_and_b32 s2, s2, 15
-; GFX9-NEXT: s_lshr_b32 s1, s1, s3
-; GFX9-NEXT: s_lshl_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s0, s0, s3
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshl_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX10-NEXT: s_andn2_b32 s3, 15, s2
+; GFX10-NEXT: s_and_b32 s3, s2, 15
+; GFX10-NEXT: s_andn2_b32 s2, 15, s2
; GFX10-NEXT: s_lshr_b32 s1, s1, 1
-; GFX10-NEXT: s_and_b32 s2, s2, 15
-; GFX10-NEXT: s_lshr_b32 s1, s1, s3
-; GFX10-NEXT: s_lshl_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_not1_b32 s3, 15, s2
+; GFX11-NEXT: s_and_b32 s3, s2, 15
+; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_b32 s2, s2, 15
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
@@ -2977,16 +3189,16 @@ define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-LABEL: s_fshl_i16_4:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 12
; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshr_b32 s1, s1, 12
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshl_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 12
; GFX9-NEXT: s_lshl_b32 s0, s0, 4
+; GFX9-NEXT: s_lshr_b32 s1, s1, 12
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -3021,16 +3233,16 @@ define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-LABEL: s_fshl_i16_5:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 11
; GFX8-NEXT: s_lshl_b32 s0, s0, 5
+; GFX8-NEXT: s_lshr_b32 s1, s1, 11
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshl_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 11
; GFX9-NEXT: s_lshl_b32 s0, s0, 5
+; GFX9-NEXT: s_lshr_b32 s1, s1, 11
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -3286,10 +3498,13 @@ define amdgpu_ps half @v_fshl_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX11-TRUE16-LABEL: v_fshl_i16_ssv:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v0.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
@@ -3426,13 +3641,13 @@ define amdgpu_ps half @v_fshl_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
;
; GFX11-TRUE16-LABEL: v_fshl_i16_vss:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, 15
; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s1, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s1, 15, s1
; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -3494,40 +3709,72 @@ define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
;
; GFX9-LABEL: s_fshl_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, 1, s1 op_sel_hi:[0,1]
-; GFX9-NEXT: s_xor_b32 s1, s2, -1
-; GFX9-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX9-NEXT: s_and_b32 s1, s2, 0xf000f
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, s1, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s3, s2, 0xf000f
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshr_b32 s5, s3, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s3
+; GFX9-NEXT: s_lshl_b32 s3, s4, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX9-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s3, s3, 0x10001
+; GFX9-NEXT: s_lshr_b32 s1, s1, 1
+; GFX9-NEXT: s_andn2_b32 s2, 0xf000f, s2
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX9-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_lshr_b32 s2, s3, s2
+; GFX9-NEXT: s_lshr_b32 s1, s1, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshl_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_lshrrev_b16 v0, 1, s1 op_sel_hi:[0,1]
-; GFX10-NEXT: s_xor_b32 s1, s2, -1
-; GFX10-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX10-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX10-NEXT: v_pk_lshlrev_b16 v1, s2, s0
-; GFX10-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s6, s1, 0xffff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10-NEXT: s_and_b32 s3, s2, 0xf000f
+; GFX10-NEXT: s_lshr_b32 s6, s6, 0x10001
+; GFX10-NEXT: s_lshr_b32 s1, s1, 1
+; GFX10-NEXT: s_andn2_b32 s2, 0xf000f, s2
+; GFX10-NEXT: s_lshr_b32 s4, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s3, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX10-NEXT: s_lshl_b32 s0, s0, s3
+; GFX10-NEXT: s_lshl_b32 s3, s4, s5
+; GFX10-NEXT: s_and_b32 s4, s1, 0xffff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10-NEXT: s_lshr_b32 s5, s2, 16
+; GFX10-NEXT: s_lshr_b32 s2, s4, s2
+; GFX10-NEXT: s_lshr_b32 s1, s1, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v2i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, 1, s1 op_sel_hi:[0,1]
-; GFX11-NEXT: s_xor_b32 s1, s2, -1
-; GFX11-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX11-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, s2, s0
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_and_b32 s6, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_and_b32 s3, s2, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s6, s6, 0x10001
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshl_b32 s3, s4, s5
+; GFX11-NEXT: s_and_b32 s4, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_lshr_b32 s5, s2, 16
+; GFX11-NEXT: s_lshr_b32 s2, s4, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to i32
@@ -3701,36 +3948,48 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
;
; GFX9-LABEL: v_fshl_v2i16_ssv:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, 1, s1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_and_b32_e32 v2, 0xf000f, v2
+; GFX9-NEXT: v_and_b32_e32 v1, 0xf000f, v0
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, v1, s0
+; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 0x10001
+; GFX9-NEXT: s_lshr_b32 s1, s1, 1
; GFX9-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, v0, s0
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_pk_lshrrev_b16 v0, v0, s0
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_v2i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX10-NEXT: v_pk_lshrrev_b16 v2, 1, s1 op_sel_hi:[0,1]
+; GFX10-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 16
; GFX10-NEXT: v_and_b32_e32 v0, 0xf000f, v0
+; GFX10-NEXT: s_lshr_b32 s2, s2, 0x10001
; GFX10-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX10-NEXT: s_lshr_b32 s1, s1, 1
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX10-NEXT: v_pk_lshlrev_b16 v0, v0, s0
-; GFX10-NEXT: v_pk_lshrrev_b16 v1, v1, v2
+; GFX10-NEXT: v_pk_lshrrev_b16 v1, v1, s1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_v2i16_ssv:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, s1 op_sel_hi:[0,1]
+; GFX11-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_lshr_b32 s2, s2, 0x10001
; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX11-NEXT: v_pk_lshlrev_b16 v0, v0, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, v1, v2
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
@@ -3776,37 +4035,46 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
;
; GFX9-LABEL: v_fshl_v2i16_svs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s2, s1, -1
+; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_andn2_b32 s1, 0xf000f, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s2, s3, s4
; GFX9-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX9-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, s2, v0
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, s1, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: v_pk_lshrrev_b16 v0, s1, v0
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_v2i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: s_xor_b32 s2, s1, -1
-; GFX10-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX10-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX10-NEXT: v_pk_lshlrev_b16 v1, s1, s0
-; GFX10-NEXT: v_pk_lshrrev_b16 v0, s2, v0
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX10-NEXT: s_andn2_b32 s1, 0xf000f, s1
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s4, s2, 16
+; GFX10-NEXT: v_pk_lshrrev_b16 v0, s1, v0
+; GFX10-NEXT: s_lshl_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s1, s3, s4
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_v2i16_svs:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_xor_b32 s2, s1, -1
-; GFX11-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX11-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, s1, s0
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, s2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, s1, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s1, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
@@ -3851,36 +4119,60 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
;
; GFX9-LABEL: v_fshl_v2i16_vss:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, 1, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_xor_b32 s0, s1, -1
-; GFX9-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, s0, v1
-; GFX9-NEXT: s_and_b32 s0, s1, 0xf000f
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, s0, v0
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, s2, v0
+; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s2, s2, 0x10001
+; GFX9-NEXT: s_lshr_b32 s0, s0, 1
+; GFX9-NEXT: s_andn2_b32 s1, 0xf000f, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshr_b32 s1, s2, s1
+; GFX9-NEXT: s_lshr_b32 s0, s0, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_v2i16_vss:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_lshrrev_b16 v1, 1, s0 op_sel_hi:[0,1]
-; GFX10-NEXT: s_xor_b32 s0, s1, -1
-; GFX10-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX10-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, s1, v0
-; GFX10-NEXT: v_pk_lshrrev_b16 v1, s0, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX10-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s3, 0x10001
+; GFX10-NEXT: s_lshr_b32 s0, s0, 1
+; GFX10-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX10-NEXT: s_andn2_b32 s1, 0xf000f, s1
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, s2, v0
+; GFX10-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX10-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_lshr_b32 s1, s2, s1
+; GFX10-NEXT: s_lshr_b32 s0, s0, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_v2i16_vss:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, 1, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_xor_b32 s0, s1, -1
-; GFX11-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX11-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, s1, v0
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, s0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s3, 0x10001
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, s2, v0
+; GFX11-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_lshr_b32 s1, s2, s1
+; GFX11-NEXT: s_lshr_b32 s0, s0, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
@@ -4023,39 +4315,81 @@ define amdgpu_ps i48 @s_fshl_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_not1_b32 s6, 15, s5
-; GFX11-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX11-NEXT: s_and_b32 s8, s5, 15
-; GFX11-NEXT: s_lshr_b32 s6, s7, s6
-; GFX11-NEXT: s_lshl_b32 s7, s1, s8
-; GFX11-NEXT: s_lshr_b32 s3, s3, 17
-; GFX11-NEXT: s_or_b32 s6, s7, s6
-; GFX11-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_and_not1_b32 s7, 15, s7
-; GFX11-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_lshl_b32 s1, s1, s5
-; GFX11-NEXT: s_and_not1_b32 s5, 15, s4
-; GFX11-NEXT: s_bfe_u32 s7, s2, 0xf0001
-; GFX11-NEXT: s_and_b32 s8, s4, 15
-; GFX11-NEXT: s_lshr_b32 s5, s7, s5
-; GFX11-NEXT: s_lshl_b32 s7, s0, s8
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshr_b32 s2, s2, 17
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s8
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-NEXT: s_or_b32 s4, s7, s5
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_fshl_v3i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s6, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s7, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s7, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s3, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, 0xf000f
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s8, 0x10001
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s5, 0xf000f, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s8, s3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_fshl_v3i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s6, 15, s5
+; GFX11-FAKE16-NEXT: s_bfe_u32 s7, s3, 0xf0001
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s5, 15
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s7, s6
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s1, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 17
+; GFX11-FAKE16-NEXT: s_or_b32 s6, s7, s6
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s7, 15, s7
+; GFX11-FAKE16-NEXT: s_bfe_u32 s5, s5, 0x40010
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s5, 15, s4
+; GFX11-FAKE16-NEXT: s_bfe_u32 s7, s2, 0xf0001
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s4, 15
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s7, s5
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s0, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 17
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s7, s5
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call <3 x i16> @llvm.fshl.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
%cast = bitcast <3 x i16> %result to i48
ret i48 %cast
@@ -4272,104 +4606,128 @@ define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
;
; GFX9-LABEL: s_fshl_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_andn2_b32 s6, 15, s5
-; GFX9-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX9-NEXT: s_lshr_b32 s6, s7, s6
-; GFX9-NEXT: s_and_b32 s7, s5, 15
-; GFX9-NEXT: s_lshl_b32 s7, s1, s7
-; GFX9-NEXT: s_or_b32 s6, s7, s6
-; GFX9-NEXT: s_lshr_b32 s7, s5, 16
-; GFX9-NEXT: s_lshr_b32 s3, s3, 17
-; GFX9-NEXT: s_andn2_b32 s7, 15, s7
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX9-NEXT: s_lshr_b32 s3, s3, s7
-; GFX9-NEXT: s_lshl_b32 s1, s1, s5
-; GFX9-NEXT: s_or_b32 s1, s1, s3
-; GFX9-NEXT: s_andn2_b32 s3, 15, s4
-; GFX9-NEXT: s_bfe_u32 s5, s2, 0xf0001
-; GFX9-NEXT: s_lshr_b32 s3, s5, s3
-; GFX9-NEXT: s_and_b32 s5, s4, 15
-; GFX9-NEXT: s_lshl_b32 s5, s0, s5
-; GFX9-NEXT: s_or_b32 s3, s5, s3
+; GFX9-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_lshr_b32 s8, s6, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s6
+; GFX9-NEXT: s_lshl_b32 s6, s7, s8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX9-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s6, s6, 0x10001
+; GFX9-NEXT: s_lshr_b32 s2, s2, 1
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX9-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s7, s4, 16
+; GFX9-NEXT: s_lshr_b32 s4, s6, s4
+; GFX9-NEXT: s_lshr_b32 s2, s2, s7
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s5, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s5
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
+; GFX9-NEXT: s_lshr_b32 s6, s2, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, s2
+; GFX9-NEXT: s_lshl_b32 s2, s5, s6
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: s_lshr_b32 s2, s2, 0x10001
+; GFX9-NEXT: s_lshr_b32 s3, s3, 1
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT: s_and_b32 s3, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
; GFX9-NEXT: s_lshr_b32 s5, s4, 16
-; GFX9-NEXT: s_lshr_b32 s2, s2, 17
-; GFX9-NEXT: s_andn2_b32 s5, 15, s5
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX9-NEXT: s_lshr_b32 s3, s3, s4
; GFX9-NEXT: s_lshr_b32 s2, s2, s5
-; GFX9-NEXT: s_lshl_b32 s0, s0, s4
-; GFX9-NEXT: s_or_b32 s0, s0, s2
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GFX9-NEXT: s_or_b32 s1, s1, s2
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshl_v4i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_andn2_b32 s6, 15, s5
-; GFX10-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX10-NEXT: s_and_b32 s8, s5, 15
-; GFX10-NEXT: s_lshr_b32 s6, s7, s6
-; GFX10-NEXT: s_lshl_b32 s7, s1, s8
-; GFX10-NEXT: s_lshr_b32 s3, s3, 17
-; GFX10-NEXT: s_or_b32 s6, s7, s6
-; GFX10-NEXT: s_lshr_b32 s7, s5, 16
-; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: s_andn2_b32 s7, 15, s7
-; GFX10-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX10-NEXT: s_lshr_b32 s3, s3, s7
-; GFX10-NEXT: s_lshl_b32 s1, s1, s5
-; GFX10-NEXT: s_andn2_b32 s5, 15, s4
-; GFX10-NEXT: s_bfe_u32 s7, s2, 0xf0001
-; GFX10-NEXT: s_and_b32 s8, s4, 15
-; GFX10-NEXT: s_lshr_b32 s5, s7, s5
-; GFX10-NEXT: s_lshl_b32 s7, s0, s8
+; GFX10-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX10-NEXT: s_lshr_b32 s2, s2, 16
+; GFX10-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX10-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX10-NEXT: s_lshr_b32 s2, s2, 1
+; GFX10-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX10-NEXT: s_lshr_b32 s7, s0, 16
+; GFX10-NEXT: s_lshr_b32 s8, s6, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX10-NEXT: s_lshl_b32 s0, s0, s6
+; GFX10-NEXT: s_lshl_b32 s6, s7, s8
+; GFX10-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX10-NEXT: s_lshr_b32 s2, s2, 16
; GFX10-NEXT: s_lshr_b32 s8, s4, 16
-; GFX10-NEXT: s_lshr_b32 s2, s2, 17
-; GFX10-NEXT: s_andn2_b32 s8, 15, s8
-; GFX10-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX10-NEXT: s_lshr_b32 s4, s7, s4
; GFX10-NEXT: s_lshr_b32 s2, s2, s8
-; GFX10-NEXT: s_lshl_b32 s0, s0, s4
-; GFX10-NEXT: s_or_b32 s4, s7, s5
+; GFX10-NEXT: s_and_b32 s8, s3, 0xffff
+; GFX10-NEXT: s_lshr_b32 s3, s3, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX10-NEXT: s_and_b32 s4, s5, 0xf000f
+; GFX10-NEXT: s_lshr_b32 s8, s8, 0x10001
+; GFX10-NEXT: s_lshr_b32 s3, s3, 1
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX10-NEXT: s_andn2_b32 s5, 0xf000f, s5
+; GFX10-NEXT: s_lshr_b32 s6, s1, 16
+; GFX10-NEXT: s_lshr_b32 s7, s4, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s3, s8, s3
+; GFX10-NEXT: s_lshl_b32 s1, s1, s4
+; GFX10-NEXT: s_lshl_b32 s4, s6, s7
+; GFX10-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX10-NEXT: s_lshr_b32 s3, s3, 16
+; GFX10-NEXT: s_lshr_b32 s7, s5, 16
+; GFX10-NEXT: s_lshr_b32 s5, s6, s5
+; GFX10-NEXT: s_lshr_b32 s3, s3, s7
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT: s_pack_ll_b32_b16 s3, s5, s3
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v4i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_not1_b32 s6, 15, s5
-; GFX11-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX11-NEXT: s_and_b32 s8, s5, 15
-; GFX11-NEXT: s_lshr_b32 s6, s7, s6
-; GFX11-NEXT: s_lshl_b32 s7, s1, s8
-; GFX11-NEXT: s_lshr_b32 s3, s3, 17
-; GFX11-NEXT: s_or_b32 s6, s7, s6
-; GFX11-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_and_not1_b32 s7, 15, s7
-; GFX11-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_lshl_b32 s1, s1, s5
-; GFX11-NEXT: s_and_not1_b32 s5, 15, s4
-; GFX11-NEXT: s_bfe_u32 s7, s2, 0xf0001
-; GFX11-NEXT: s_and_b32 s8, s4, 15
-; GFX11-NEXT: s_lshr_b32 s5, s7, s5
-; GFX11-NEXT: s_lshl_b32 s7, s0, s8
+; GFX11-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX11-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-NEXT: s_lshr_b32 s8, s6, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_lshl_b32 s6, s7, s8
+; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshr_b32 s2, s2, 17
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s8
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX11-NEXT: s_lshr_b32 s4, s7, s4
; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-NEXT: s_or_b32 s4, s7, s5
+; GFX11-NEXT: s_and_b32 s8, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s4, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s8, s8, 0x10001
+; GFX11-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s5, 0xf000f, s5
+; GFX11-NEXT: s_lshr_b32 s6, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s8, s3
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s7
+; GFX11-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-NEXT: s_lshr_b32 s5, s6, s5
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
; GFX11-NEXT: s_or_b32 s0, s0, s2
; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call <4 x i16> @llvm.fshl.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <2 x i32>
@@ -4741,19 +5099,19 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
; GFX6-LABEL: v_fshl_i64_48:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[2:3], 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_i64_48:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b64 v[4:5], 16, v[2:3]
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v0, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshl_i64_48:
@@ -4787,39 +5145,43 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64 %amt) {
; GFX6-LABEL: v_fshl_i64_ssv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshl_b64 v[2:3], s[0:1], v0
+; GFX6-NEXT: v_and_b32_e32 v1, 63, v0
+; GFX6-NEXT: v_lshl_b64 v[1:2], s[0:1], v1
; GFX6-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; GFX6-NEXT: v_not_b32_e32 v0, v0
-; GFX6-NEXT: v_lshr_b64 v[0:1], s[0:1], v0
-; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_bfi_b32 v0, v0, 0, 63
+; GFX6-NEXT: v_lshr_b64 v[3:4], s[0:1], v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v3
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v4
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i64_ssv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], v0, s[0:1]
+; GFX8-NEXT: v_and_b32_e32 v1, 63, v0
+; GFX8-NEXT: v_lshlrev_b64 v[1:2], v1, s[0:1]
; GFX8-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; GFX8-NEXT: v_not_b32_e32 v0, v0
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, 63
+; GFX8-NEXT: v_lshrrev_b64 v[3:4], v0, s[0:1]
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i64_ssv:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], v0, s[0:1]
+; GFX9-NEXT: v_and_b32_e32 v1, 63, v0
+; GFX9-NEXT: v_lshlrev_b64 v[1:2], v1, s[0:1]
; GFX9-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; GFX9-NEXT: v_not_b32_e32 v0, v0
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
-; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_bfi_b32 v0, v0, 0, 63
+; GFX9-NEXT: v_lshrrev_b64 v[3:4], v0, s[0:1]
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v2, v4
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i64_ssv:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_not_b32_e32 v2, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 63, v0
+; GFX10-NEXT: v_bfi_b32 v2, v0, 0, 63
; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], v0, s[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
; GFX10-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
@@ -4827,13 +5189,14 @@ define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
;
; GFX11-LABEL: v_fshl_i64_ssv:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_not_b32_e32 v2, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 63, v0
+; GFX11-NEXT: v_bfi_b32 v2, v0, 0, 63
; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, s[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
@@ -4845,53 +5208,62 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX6-LABEL: v_fshl_i64_svs:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], 1
-; GFX6-NEXT: s_not_b32 s3, s2
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s3
; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: s_andn2_b32 s2, 63, s2
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i64_svs:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX8-NEXT: s_not_b32 s3, s2
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: s_andn2_b32 s2, 63, s2
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i64_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX9-NEXT: s_not_b32 s3, s2
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: s_andn2_b32 s2, 63, s2
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i64_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX10-NEXT: s_not_b32 s3, s2
+; GFX10-NEXT: s_andn2_b32 s3, 63, s2
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
-; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i64_svs:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -4901,55 +5273,67 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg %amt) {
; GFX6-LABEL: v_fshl_i64_vss:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], s2
+; GFX6-NEXT: s_and_b32 s3, s2, 63
; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
; GFX6-NEXT: s_not_b32 s2, s2
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], s3
; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i64_vss:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX8-NEXT: s_and_b32 s3, s2, 63
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
; GFX8-NEXT: s_not_b32 s2, s2
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i64_vss:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX9-NEXT: s_and_b32 s3, s2, 63
; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
; GFX9-NEXT: s_not_b32 s2, s2
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i64_vss:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_not_b32 s2, s2
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT: s_not_b32 s3, s2
+; GFX10-NEXT: s_and_b32 s2, s2, 63
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i64_vss:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 63
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -5255,130 +5639,241 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX6-LABEL: v_fshl_i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX6-NEXT: v_mov_b32_e32 v10, 0
-; GFX6-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10]
-; GFX6-NEXT: v_not_b32_e32 v13, v8
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX6-NEXT: v_lshr_b64 v[4:5], v[4:5], 1
-; GFX6-NEXT: v_cndmask_b32_e32 v10, v1, v7, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v0, v6, vcc
-; GFX6-NEXT: v_lshl_b64 v[11:12], v[9:10], v8
-; GFX6-NEXT: v_lshr_b64 v[5:6], v[4:5], v13
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[9:10], 1
-; GFX6-NEXT: v_or_b32_e32 v4, v12, v6
-; GFX6-NEXT: v_lshl_b64 v[6:7], v[0:1], v8
-; GFX6-NEXT: v_lshr_b64 v[1:2], v[2:3], v13
-; GFX6-NEXT: v_or_b32_e32 v0, v11, v5
-; GFX6-NEXT: v_or_b32_e32 v3, v7, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v6, v1
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: v_and_b32_e32 v16, 0x7f, v8
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, 64, v16
+; GFX6-NEXT: v_add_i32_e32 v18, vcc, 0xffffffc0, v16
+; GFX6-NEXT: v_lshr_b64 v[9:10], v[0:1], v9
+; GFX6-NEXT: v_lshl_b64 v[11:12], v[2:3], v16
+; GFX6-NEXT: v_lshl_b64 v[13:14], v[0:1], v16
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], v18
+; GFX6-NEXT: v_or_b32_e32 v9, v9, v11
+; GFX6-NEXT: v_or_b32_e32 v10, v10, v12
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX6-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v12, 0, v14, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX6-NEXT: v_cndmask_b32_e32 v10, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v13, v1, v3, vcc
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[4:5], 1
+; GFX6-NEXT: v_mov_b32_e32 v15, 0x7f
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 31, v6
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[6:7], 1
+; GFX6-NEXT: v_bfi_b32 v14, v8, 0, v15
+; GFX6-NEXT: v_not_b32_e32 v17, 63
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, 64, v14
+; GFX6-NEXT: v_add_i32_e32 v15, vcc, v14, v17
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[0:1], v14
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[2:3], v6
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[2:3], v14
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], v15
+; GFX6-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v11, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v12, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v10, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v13, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX8-NEXT: v_mov_b32_e32 v10, 0
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10]
-; GFX8-NEXT: v_not_b32_e32 v13, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v10, v1, v7, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v0, v6, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[11:12], v8, v[9:10]
-; GFX8-NEXT: v_lshrrev_b64 v[5:6], v13, v[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[9:10]
-; GFX8-NEXT: v_or_b32_e32 v4, v12, v6
-; GFX8-NEXT: v_lshlrev_b64 v[6:7], v8, v[0:1]
-; GFX8-NEXT: v_lshrrev_b64 v[1:2], v13, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v0, v11, v5
-; GFX8-NEXT: v_or_b32_e32 v3, v7, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v6, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: v_and_b32_e32 v16, 0x7f, v8
+; GFX8-NEXT: v_sub_u32_e32 v9, vcc, 64, v16
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, 0xffffffc0, v16
+; GFX8-NEXT: v_lshrrev_b64 v[9:10], v9, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[11:12], v16, v[2:3]
+; GFX8-NEXT: v_lshlrev_b64 v[13:14], v16, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v18, v[0:1]
+; GFX8-NEXT: v_or_b32_e32 v9, v9, v11
+; GFX8-NEXT: v_or_b32_e32 v10, v10, v12
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v12, 0, v14, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v1, v3, vcc
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v15, 0x7f
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 31, v6
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[6:7]
+; GFX8-NEXT: v_bfi_b32 v14, v8, 0, v15
+; GFX8-NEXT: v_not_b32_e32 v17, 63
+; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 64, v14
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v14, v17
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v14, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], v14, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v15, v[2:3]
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v11, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v12, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v10, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v13, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshl_i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX9-NEXT: v_mov_b32_e32 v10, 0
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10]
-; GFX9-NEXT: v_not_b32_e32 v13, v8
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v10, v1, v7, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v0, v6, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[11:12], v8, v[9:10]
-; GFX9-NEXT: v_lshrrev_b64 v[5:6], v13, v[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], 1, v[9:10]
-; GFX9-NEXT: v_or_b32_e32 v4, v12, v6
-; GFX9-NEXT: v_lshlrev_b64 v[6:7], v8, v[0:1]
-; GFX9-NEXT: v_lshrrev_b64 v[1:2], v13, v[2:3]
-; GFX9-NEXT: v_or_b32_e32 v0, v11, v5
-; GFX9-NEXT: v_or_b32_e32 v3, v7, v2
-; GFX9-NEXT: v_or_b32_e32 v2, v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NEXT: v_and_b32_e32 v16, 0x7f, v8
+; GFX9-NEXT: v_sub_u32_e32 v9, 64, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0xffffffc0, v16
+; GFX9-NEXT: v_lshrrev_b64 v[9:10], v9, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[11:12], v16, v[2:3]
+; GFX9-NEXT: v_lshlrev_b64 v[13:14], v16, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v17, v[0:1]
+; GFX9-NEXT: v_or_b32_e32 v9, v9, v11
+; GFX9-NEXT: v_or_b32_e32 v10, v10, v12
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v12, 0, v14, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v1, v10, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-NEXT: v_mov_b32_e32 v15, 0x7f
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v0, v2, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], 1, v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v9, v3, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], 1, v[6:7]
+; GFX9-NEXT: v_bfi_b32 v14, v8, 0, v15
+; GFX9-NEXT: v_lshl_or_b32 v1, v6, 31, v1
+; GFX9-NEXT: v_sub_u32_e32 v6, 64, v14
+; GFX9-NEXT: v_add_u32_e32 v15, 0xffffffc0, v14
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], v14, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3]
+; GFX9-NEXT: v_lshrrev_b64 v[8:9], v14, v[2:3]
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], v15, v[2:3]
+; GFX9-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, v11, v0
+; GFX9-NEXT: v_or_b32_e32 v1, v12, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v10, v2
+; GFX9-NEXT: v_or_b32_e32 v3, v13, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fshl_i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX10-NEXT: v_mov_b32_e32 v10, 0
-; GFX10-NEXT: v_not_b32_e32 v11, v8
-; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[9:10]
-; GFX10-NEXT: v_cndmask_b32_e32 v10, v1, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v0, v6, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT: v_lshrrev_b64 v[3:4], 1, v[4:5]
-; GFX10-NEXT: v_lshrrev_b64 v[5:6], 1, v[9:10]
-; GFX10-NEXT: v_lshlrev_b64 v[9:10], v8, v[9:10]
-; GFX10-NEXT: v_lshlrev_b64 v[7:8], v8, v[0:1]
-; GFX10-NEXT: v_lshrrev_b64 v[2:3], v11, v[3:4]
-; GFX10-NEXT: v_lshrrev_b64 v[4:5], v11, v[5:6]
-; GFX10-NEXT: v_or_b32_e32 v0, v9, v2
-; GFX10-NEXT: v_or_b32_e32 v1, v10, v3
-; GFX10-NEXT: v_or_b32_e32 v2, v7, v4
-; GFX10-NEXT: v_or_b32_e32 v3, v8, v5
+; GFX10-NEXT: v_and_b32_e32 v17, 0x7f, v8
+; GFX10-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX10-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
+; GFX10-NEXT: v_lshrrev_b64 v[9:10], 1, v[6:7]
+; GFX10-NEXT: v_sub_nc_u32_e32 v11, 64, v17
+; GFX10-NEXT: v_lshlrev_b64 v[7:8], v17, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v15, 0xffffffc0, v17
+; GFX10-NEXT: v_lshl_or_b32 v5, v6, 31, v5
+; GFX10-NEXT: v_sub_nc_u32_e32 v16, 64, v18
+; GFX10-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[13:14], v17, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v17
+; GFX10-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v18
+; GFX10-NEXT: v_lshlrev_b64 v[15:16], v16, v[9:10]
+; GFX10-NEXT: v_or_b32_e32 v11, v11, v7
+; GFX10-NEXT: v_lshrrev_b64 v[6:7], v18, v[4:5]
+; GFX10-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX10-NEXT: v_cmp_gt_u32_e64 s5, 64, v18
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v17
+; GFX10-NEXT: v_cndmask_b32_e32 v20, v0, v11, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[11:12], v19, v[9:10]
+; GFX10-NEXT: v_or_b32_e32 v0, v6, v15
+; GFX10-NEXT: v_or_b32_e32 v6, v7, v16
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v8, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v18
+; GFX10-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v11, v0, s5
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v18, v[9:10]
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v12, v6, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v9, 0, v14, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v20, v2, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, v4, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v6, v5, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, v0, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, v1, s5
+; GFX10-NEXT: v_or_b32_e32 v0, v13, v4
+; GFX10-NEXT: v_or_b32_e32 v1, v9, v5
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshl_i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_and_b32 v9, 64, v8
-; GFX11-NEXT: v_not_b32_e32 v11, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[9:10]
-; GFX11-NEXT: v_dual_cndmask_b32 v10, v1, v7 :: v_dual_cndmask_b32 v5, v7, v5
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v9, v0, v6
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 1, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[5:6], 1, v[9:10]
-; GFX11-NEXT: v_lshlrev_b64 v[9:10], v8, v[9:10]
-; GFX11-NEXT: v_lshlrev_b64 v[7:8], v8, v[0:1]
+; GFX11-NEXT: v_and_b32_e32 v17, 0x7f, v8
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX11-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
+; GFX11-NEXT: v_lshrrev_b64 v[9:10], 1, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v11, 64, v17
+; GFX11-NEXT: v_lshlrev_b64 v[7:8], v17, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v15, 0xffffffc0, v17
+; GFX11-NEXT: v_lshl_or_b32 v5, v6, 31, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v18
+; GFX11-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[13:14], v17, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v17
+; GFX11-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v18
+; GFX11-NEXT: v_lshlrev_b64 v[15:16], v16, v[9:10]
+; GFX11-NEXT: v_or_b32_e32 v11, v11, v7
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], v18, v[4:5]
+; GFX11-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v18
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v0, v11, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[11:12], v19, v[9:10]
+; GFX11-NEXT: v_or_b32_e32 v0, v6, v15
+; GFX11-NEXT: v_or_b32_e32 v6, v7, v16
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v8, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v17
+; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v11, v0, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v18, v[9:10]
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, v6, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v9, 0, v14, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v20, v2, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, v4, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v5, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v7, 0, v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v11, v[3:4]
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], v11, v[5:6]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v9, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v10, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v7, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v5
+; GFX11-NEXT: v_or_b32_e32 v0, v13, v4
+; GFX11-NEXT: v_or_b32_e32 v1, v9, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
@@ -5552,139 +6047,288 @@ define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 inreg %amt) {
; GFX6-LABEL: v_fshl_i128_svs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s6, s4, 64
-; GFX6-NEXT: s_mov_b32 s7, 0
-; GFX6-NEXT: v_cmp_ne_u64_e64 vcc, s[6:7], 0
-; GFX6-NEXT: v_mov_b32_e32 v4, s1
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v4, v3, vcc
-; GFX6-NEXT: v_mov_b32_e32 v4, s0
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX6-NEXT: s_sub_i32 s12, s5, 64
+; GFX6-NEXT: s_sub_i32 s8, 64, s5
+; GFX6-NEXT: s_cmp_lt_u32 s5, 64
+; GFX6-NEXT: s_cselect_b32 s13, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s5, 0
+; GFX6-NEXT: s_cselect_b32 s5, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[8:9], s[0:1], s8
+; GFX6-NEXT: s_lshl_b64 s[10:11], s[2:3], s4
+; GFX6-NEXT: s_lshl_b64 s[6:7], s[0:1], s4
+; GFX6-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX6-NEXT: s_cmp_lg_u32 s13, 0
+; GFX6-NEXT: s_cselect_b64 s[6:7], s[6:7], 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX6-NEXT: s_cmp_lg_u32 s5, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], 1
-; GFX6-NEXT: s_not_b32 s5, s4
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[4:5], 1
-; GFX6-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX6-NEXT: v_lshl_b64 v[6:7], v[4:5], s4
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s5
-; GFX6-NEXT: s_cselect_b32 s1, s1, s3
-; GFX6-NEXT: s_cselect_b32 s0, s0, s2
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], s5
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX6-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX6-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX6-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX6-NEXT: s_andn2_b32 s2, 0x7f, s4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 31, v2
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], 1
+; GFX6-NEXT: s_sub_i32 s3, s2, 64
+; GFX6-NEXT: s_sub_i32 s4, 64, s2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT: s_cmp_lt_u32 s2, 64
+; GFX6-NEXT: s_cselect_b32 s5, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s2, 0
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[0:1], s2
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[2:3], s4
+; GFX6-NEXT: s_cselect_b32 s8, 1, 0
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[2:3], s2
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], s3
+; GFX6-NEXT: s_cmp_lg_u32 s5, 0
+; GFX6-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: s_cmp_lg_u32 s8, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: s_cmp_lg_u32 s5, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: v_mov_b32_e32 v0, s6
+; GFX6-NEXT: v_mov_b32_e32 v1, s7
+; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v5
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i128_svs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_not_b32 s5, s4
-; GFX8-NEXT: s_and_b32 s6, s4, 64
-; GFX8-NEXT: s_mov_b32 s7, 0
-; GFX8-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v4, s1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v4, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX8-NEXT: s_sub_i32 s12, s5, 64
+; GFX8-NEXT: s_sub_i32 s8, 64, s5
+; GFX8-NEXT: s_cmp_lt_u32 s5, 64
+; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s5, 0
+; GFX8-NEXT: s_cselect_b32 s5, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[8:9], s[0:1], s8
+; GFX8-NEXT: s_lshl_b64 s[10:11], s[2:3], s4
+; GFX8-NEXT: s_lshl_b64 s[6:7], s[0:1], s4
+; GFX8-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX8-NEXT: s_cmp_lg_u32 s13, 0
+; GFX8-NEXT: s_cselect_b64 s[6:7], s[6:7], 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX8-NEXT: s_cmp_lg_u32 s5, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], s4, v[4:5]
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], s5, v[0:1]
-; GFX8-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[4:5]
-; GFX8-NEXT: s_cselect_b32 s1, s1, s3
-; GFX8-NEXT: s_cselect_b32 s0, s0, s2
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], s5, v[2:3]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX8-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX8-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX8-NEXT: s_andn2_b32 s2, 0x7f, s4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 31, v2
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX8-NEXT: s_sub_i32 s3, s2, 64
+; GFX8-NEXT: s_sub_i32 s4, 64, s2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT: s_cmp_lt_u32 s2, 64
+; GFX8-NEXT: s_cselect_b32 s5, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s2, 0
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX8-NEXT: s_cselect_b32 s8, 1, 0
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX8-NEXT: s_cmp_lg_u32 s5, 0
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: s_cmp_lg_u32 s8, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: s_cmp_lg_u32 s5, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i128_svs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_not_b32 s5, s4
-; GFX9-NEXT: s_and_b32 s6, s4, 64
-; GFX9-NEXT: s_mov_b32 s7, 0
-; GFX9-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX9-NEXT: s_sub_i32 s12, s5, 64
+; GFX9-NEXT: s_sub_i32 s8, 64, s5
+; GFX9-NEXT: s_cmp_lt_u32 s5, 64
+; GFX9-NEXT: s_cselect_b32 s13, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s5, 0
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[0:1], s8
+; GFX9-NEXT: s_lshl_b64 s[10:11], s[2:3], s4
+; GFX9-NEXT: s_lshl_b64 s[6:7], s[0:1], s4
+; GFX9-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[6:7], 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[8:9], s[0:1]
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], s4, v[4:5]
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], s5, v[0:1]
-; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], 1, v[4:5]
-; GFX9-NEXT: s_cselect_b32 s1, s1, s3
-; GFX9-NEXT: s_cselect_b32 s0, s0, s2
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], s5, v[2:3]
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX9-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX9-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX9-NEXT: s_cmp_lg_u32 s5, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT: s_andn2_b32 s2, 0x7f, s4
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 31, v1
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX9-NEXT: s_sub_i32 s3, s2, 64
+; GFX9-NEXT: s_sub_i32 s4, 64, s2
+; GFX9-NEXT: s_cmp_lt_u32 s2, 64
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s2, 0
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX9-NEXT: s_cselect_b32 s8, 1, 0
+; GFX9-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX9-NEXT: s_cmp_lg_u32 s5, 0
+; GFX9-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_cmp_lg_u32 s8, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i128_svs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s6, s4, 64
-; GFX10-NEXT: s_mov_b32 s7, 0
-; GFX10-NEXT: s_not_b32 s5, s4
-; GFX10-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s1, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s0, v2, vcc_lo
-; GFX10-NEXT: s_and_b32 s6, vcc_lo, exec_lo
-; GFX10-NEXT: s_cselect_b32 s1, s1, s3
-; GFX10-NEXT: s_cselect_b32 s0, s0, s2
+; GFX10-NEXT: s_and_b32 s5, s4, 0x7f
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX10-NEXT: v_lshrrev_b64 v[5:6], 1, v[3:4]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], s4, v[3:4]
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], s5, v[0:1]
-; GFX10-NEXT: v_lshrrev_b64 v[4:5], s5, v[5:6]
-; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX10-NEXT: v_or_b32_e32 v2, s0, v4
-; GFX10-NEXT: v_or_b32_e32 v3, s1, v5
+; GFX10-NEXT: s_sub_i32 s12, s5, 64
+; GFX10-NEXT: s_sub_i32 s6, 64, s5
+; GFX10-NEXT: s_cmp_lt_u32 s5, 64
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: v_lshl_or_b32 v1, v2, 31, v1
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], s4
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[0:1], s4
+; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT: s_andn2_b32 s2, 0x7f, s4
+; GFX10-NEXT: s_sub_i32 s4, 64, s2
+; GFX10-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX10-NEXT: s_sub_i32 s3, s2, 64
+; GFX10-NEXT: s_cmp_lt_u32 s2, 64
+; GFX10-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX10-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s2, 0
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i128_svs:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s4, 64
-; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_not_b32 s5, s4
-; GFX11-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-NEXT: v_dual_cndmask_b32 v4, s1, v3 :: v_dual_cndmask_b32 v1, v3, v1
-; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v3, s0, v2
-; GFX11-NEXT: s_and_b32 s6, vcc_lo, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, s1, s3
-; GFX11-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_and_b32 s5, s4, 0x7f
; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[5:6], 1, v[3:4]
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], s4, v[3:4]
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s5, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], s5, v[5:6]
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-NEXT: v_or_b32_e32 v2, s0, v4
+; GFX11-NEXT: s_sub_i32 s12, s5, 64
+; GFX11-NEXT: s_sub_i32 s6, 64, s5
+; GFX11-NEXT: s_cmp_lt_u32 s5, 64
+; GFX11-NEXT: s_cselect_b32 s13, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: v_lshl_or_b32 v1, v2, 31, v1
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s4
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[0:1], s4
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-NEXT: s_and_not1_b32 s2, 0x7f, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s4, 64, s2
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX11-NEXT: s_sub_i32 s3, s2, 64
+; GFX11-NEXT: s_cmp_lt_u32 s2, 64
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, s1, v5
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
@@ -6178,226 +6822,449 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX6-LABEL: v_fshl_v2i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX6-NEXT: v_mov_b32_e32 v18, 0
-; GFX6-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[17:18]
-; GFX6-NEXT: v_cndmask_b32_e32 v22, v1, v11, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v21, v0, v10, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[8:9], 1
-; GFX6-NEXT: v_lshl_b64 v[10:11], v[0:1], v16
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[21:22], 1
-; GFX6-NEXT: v_lshl_b64 v[2:3], v[21:22], v16
-; GFX6-NEXT: v_not_b32_e32 v16, v16
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[8:9], v16
-; GFX6-NEXT: v_lshr_b64 v[16:17], v[0:1], v16
-; GFX6-NEXT: v_or_b32_e32 v1, v3, v9
-; GFX6-NEXT: v_or_b32_e32 v3, v11, v17
-; GFX6-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX6-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[17:18]
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v8
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v15, v13, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v18, v5, v15, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v17, v4, v14, vcc
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[8:9], 1
-; GFX6-NEXT: v_not_b32_e32 v13, v20
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[17:18], 1
-; GFX6-NEXT: v_lshl_b64 v[21:22], v[17:18], v20
-; GFX6-NEXT: v_lshr_b64 v[11:12], v[8:9], v13
-; GFX6-NEXT: v_or_b32_e32 v2, v10, v16
-; GFX6-NEXT: v_lshl_b64 v[9:10], v[4:5], v20
-; GFX6-NEXT: v_lshr_b64 v[5:6], v[6:7], v13
-; GFX6-NEXT: v_or_b32_e32 v8, v22, v12
-; GFX6-NEXT: v_or_b32_e32 v4, v21, v11
-; GFX6-NEXT: v_or_b32_e32 v7, v10, v6
-; GFX6-NEXT: v_or_b32_e32 v6, v9, v5
-; GFX6-NEXT: v_mov_b32_e32 v5, v8
+; GFX6-NEXT: v_and_b32_e32 v19, 0x7f, v16
+; GFX6-NEXT: v_not_b32_e32 v18, 63
+; GFX6-NEXT: v_sub_i32_e32 v23, vcc, 64, v19
+; GFX6-NEXT: v_add_i32_e32 v27, vcc, v19, v18
+; GFX6-NEXT: v_lshr_b64 v[23:24], v[0:1], v23
+; GFX6-NEXT: v_lshl_b64 v[25:26], v[2:3], v19
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[0:1], v19
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], v27
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX6-NEXT: v_or_b32_e32 v19, v23, v25
+; GFX6-NEXT: v_or_b32_e32 v23, v24, v26
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v19, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v23, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[8:9], 1
+; GFX6-NEXT: v_mov_b32_e32 v17, 0x7f
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 31, v10
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[10:11], 1
+; GFX6-NEXT: v_bfi_b32 v10, v16, 0, v17
+; GFX6-NEXT: v_cndmask_b32_e32 v24, 0, v21, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v25, 0, v22, vcc
+; GFX6-NEXT: v_add_i32_e32 v16, vcc, v10, v18
+; GFX6-NEXT: v_sub_i32_e32 v21, vcc, 64, v10
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[2:3], v10
+; GFX6-NEXT: v_lshr_b64 v[10:11], v[0:1], v10
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[2:3], v21
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], v16
+; GFX6-NEXT: v_or_b32_e32 v10, v10, v21
+; GFX6-NEXT: v_or_b32_e32 v11, v11, v22
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX6-NEXT: v_and_b32_e32 v16, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v16
+; GFX6-NEXT: v_lshr_b64 v[10:11], v[4:5], v10
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[6:7], v16
+; GFX6-NEXT: v_or_b32_e32 v2, v19, v2
+; GFX6-NEXT: v_add_i32_e32 v19, vcc, v16, v18
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], v16
+; GFX6-NEXT: v_or_b32_e32 v16, v10, v21
+; GFX6-NEXT: v_or_b32_e32 v21, v11, v22
+; GFX6-NEXT: v_lshl_b64 v[10:11], v[4:5], v19
+; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v8, v10, v16, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v9, v11, v21, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[12:13], 1
+; GFX6-NEXT: v_lshlrev_b32_e32 v10, 31, v14
+; GFX6-NEXT: v_or_b32_e32 v9, v9, v10
+; GFX6-NEXT: v_lshr_b64 v[10:11], v[14:15], 1
+; GFX6-NEXT: v_bfi_b32 v14, v20, 0, v17
+; GFX6-NEXT: v_add_i32_e32 v18, vcc, v14, v18
+; GFX6-NEXT: v_sub_i32_e32 v16, vcc, 64, v14
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX6-NEXT: v_lshr_b64 v[12:13], v[10:11], v14
+; GFX6-NEXT: v_lshr_b64 v[14:15], v[8:9], v14
+; GFX6-NEXT: v_lshl_b64 v[16:17], v[10:11], v16
+; GFX6-NEXT: v_lshr_b64 v[10:11], v[10:11], v18
+; GFX6-NEXT: v_or_b32_e32 v14, v14, v16
+; GFX6-NEXT: v_or_b32_e32 v15, v15, v17
+; GFX6-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v8, v10, v8, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v9, v11, v9, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v24, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v25, v1
+; GFX6-NEXT: v_or_b32_e32 v3, v23, v3
+; GFX6-NEXT: v_or_b32_e32 v4, v4, v8
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v9
+; GFX6-NEXT: v_or_b32_e32 v6, v6, v10
+; GFX6-NEXT: v_or_b32_e32 v7, v7, v11
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX8-NEXT: v_mov_b32_e32 v18, 0
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[17:18]
-; GFX8-NEXT: v_cndmask_b32_e32 v22, v1, v11, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v21, v0, v10, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[10:11], v16, v[0:1]
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[21:22]
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], v16, v[21:22]
-; GFX8-NEXT: v_not_b32_e32 v16, v16
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], v16, v[8:9]
-; GFX8-NEXT: v_lshrrev_b64 v[16:17], v16, v[0:1]
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v11, v17
-; GFX8-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[17:18]
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v15, v13, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v18, v5, v15, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v17, v4, v14, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX8-NEXT: v_not_b32_e32 v13, v20
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], 1, v[17:18]
-; GFX8-NEXT: v_lshlrev_b64 v[21:22], v20, v[17:18]
-; GFX8-NEXT: v_lshrrev_b64 v[11:12], v13, v[8:9]
-; GFX8-NEXT: v_or_b32_e32 v2, v10, v16
-; GFX8-NEXT: v_lshlrev_b64 v[9:10], v20, v[4:5]
-; GFX8-NEXT: v_lshrrev_b64 v[5:6], v13, v[6:7]
-; GFX8-NEXT: v_or_b32_e32 v8, v22, v12
-; GFX8-NEXT: v_or_b32_e32 v4, v21, v11
-; GFX8-NEXT: v_or_b32_e32 v7, v10, v6
-; GFX8-NEXT: v_or_b32_e32 v6, v9, v5
-; GFX8-NEXT: v_mov_b32_e32 v5, v8
+; GFX8-NEXT: v_and_b32_e32 v19, 0x7f, v16
+; GFX8-NEXT: v_not_b32_e32 v18, 63
+; GFX8-NEXT: v_sub_u32_e32 v23, vcc, 64, v19
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, v19, v18
+; GFX8-NEXT: v_lshrrev_b64 v[23:24], v23, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[25:26], v19, v[2:3]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v19, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v27, v[0:1]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX8-NEXT: v_or_b32_e32 v19, v23, v25
+; GFX8-NEXT: v_or_b32_e32 v23, v24, v26
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v19, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v23, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[8:9]
+; GFX8-NEXT: v_mov_b32_e32 v17, 0x7f
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 31, v10
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[10:11]
+; GFX8-NEXT: v_bfi_b32 v10, v16, 0, v17
+; GFX8-NEXT: v_cndmask_b32_e32 v24, 0, v21, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v25, 0, v22, vcc
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v10, v18
+; GFX8-NEXT: v_sub_u32_e32 v21, vcc, 64, v10
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], v10, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v21, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v16, v[2:3]
+; GFX8-NEXT: v_or_b32_e32 v10, v10, v21
+; GFX8-NEXT: v_or_b32_e32 v11, v11, v22
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX8-NEXT: v_and_b32_e32 v16, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v16
+; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v16, v[6:7]
+; GFX8-NEXT: v_or_b32_e32 v2, v19, v2
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, v16, v18
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], v16, v[4:5]
+; GFX8-NEXT: v_or_b32_e32 v16, v10, v21
+; GFX8-NEXT: v_or_b32_e32 v21, v11, v22
+; GFX8-NEXT: v_lshlrev_b64 v[10:11], v19, v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v16, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v11, v21, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
+; GFX8-NEXT: v_lshlrev_b32_e32 v10, 31, v14
+; GFX8-NEXT: v_or_b32_e32 v9, v9, v10
+; GFX8-NEXT: v_lshrrev_b64 v[10:11], 1, v[14:15]
+; GFX8-NEXT: v_bfi_b32 v14, v20, 0, v17
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, v14, v18
+; GFX8-NEXT: v_sub_u32_e32 v16, vcc, 64, v14
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX8-NEXT: v_lshrrev_b64 v[12:13], v14, v[10:11]
+; GFX8-NEXT: v_lshrrev_b64 v[14:15], v14, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[10:11]
+; GFX8-NEXT: v_lshrrev_b64 v[10:11], v18, v[10:11]
+; GFX8-NEXT: v_or_b32_e32 v14, v14, v16
+; GFX8-NEXT: v_or_b32_e32 v15, v15, v17
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v8, v10, v8, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v9, v11, v9, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v24, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v25, v1
+; GFX8-NEXT: v_or_b32_e32 v3, v23, v3
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v8
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v9
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v10
+; GFX8-NEXT: v_or_b32_e32 v7, v7, v11
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshl_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX9-NEXT: v_mov_b32_e32 v18, 0
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[17:18]
-; GFX9-NEXT: v_cndmask_b32_e32 v22, v1, v11, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v21, v0, v10, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_and_b32_e32 v23, 0x7f, v16
; GFX9-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX9-NEXT: v_lshlrev_b64 v[10:11], v16, v[0:1]
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], 1, v[21:22]
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], v16, v[21:22]
-; GFX9-NEXT: v_not_b32_e32 v16, v16
-; GFX9-NEXT: v_lshrrev_b64 v[8:9], v16, v[8:9]
-; GFX9-NEXT: v_lshrrev_b64 v[16:17], v16, v[0:1]
-; GFX9-NEXT: v_or_b32_e32 v1, v3, v9
-; GFX9-NEXT: v_or_b32_e32 v3, v11, v17
-; GFX9-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[17:18]
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v8
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v15, v13, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v5, v15, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v4, v14, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX9-NEXT: v_not_b32_e32 v13, v20
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[6:7], 1, v[17:18]
-; GFX9-NEXT: v_lshlrev_b64 v[21:22], v20, v[17:18]
-; GFX9-NEXT: v_lshrrev_b64 v[11:12], v13, v[8:9]
-; GFX9-NEXT: v_or_b32_e32 v2, v10, v16
-; GFX9-NEXT: v_lshlrev_b64 v[9:10], v20, v[4:5]
-; GFX9-NEXT: v_lshrrev_b64 v[5:6], v13, v[6:7]
-; GFX9-NEXT: v_or_b32_e32 v8, v22, v12
-; GFX9-NEXT: v_or_b32_e32 v4, v21, v11
-; GFX9-NEXT: v_or_b32_e32 v7, v10, v6
-; GFX9-NEXT: v_or_b32_e32 v6, v9, v5
-; GFX9-NEXT: v_mov_b32_e32 v5, v8
+; GFX9-NEXT: v_sub_u32_e32 v17, 64, v23
+; GFX9-NEXT: v_mov_b32_e32 v24, 0x7f
+; GFX9-NEXT: v_lshrrev_b64 v[17:18], v17, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[21:22], v23, v[2:3]
+; GFX9-NEXT: v_lshl_or_b32 v9, v10, 31, v9
+; GFX9-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX9-NEXT: v_bfi_b32 v25, v16, 0, v24
+; GFX9-NEXT: v_sub_u32_e32 v16, 64, v25
+; GFX9-NEXT: v_or_b32_e32 v21, v17, v21
+; GFX9-NEXT: v_or_b32_e32 v22, v18, v22
+; GFX9-NEXT: v_lshlrev_b64 v[16:17], v16, v[10:11]
+; GFX9-NEXT: v_lshrrev_b64 v[18:19], v25, v[8:9]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
+; GFX9-NEXT: v_or_b32_e32 v18, v18, v16
+; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v23
+; GFX9-NEXT: v_or_b32_e32 v19, v19, v17
+; GFX9-NEXT: v_lshlrev_b64 v[16:17], v16, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v23, v[0:1]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
+; GFX9-NEXT: v_cndmask_b32_e32 v26, 0, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v21, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v22, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v17, v0, v2, s[4:5]
+; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v25
+; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v3, s[4:5]
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], v0, v[10:11]
+; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v25
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v1, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v25, v[10:11]
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v25
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, v26, v2
+; GFX9-NEXT: v_or_b32_e32 v2, v17, v8
+; GFX9-NEXT: v_and_b32_e32 v17, 0x7f, v20
+; GFX9-NEXT: v_cndmask_b32_e64 v19, 0, v1, s[4:5]
+; GFX9-NEXT: v_or_b32_e32 v1, v18, v3
+; GFX9-NEXT: v_sub_u32_e32 v3, 64, v17
+; GFX9-NEXT: v_lshrrev_b64 v[8:9], v3, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[10:11], v17, v[6:7]
+; GFX9-NEXT: v_or_b32_e32 v3, v16, v19
+; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v17
+; GFX9-NEXT: v_or_b32_e32 v10, v8, v10
+; GFX9-NEXT: v_or_b32_e32 v11, v9, v11
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], v17, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v5, v11, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v4, v6, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], 1, v[12:13]
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v8, v7, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[6:7], 1, v[14:15]
+; GFX9-NEXT: v_bfi_b32 v13, v20, 0, v24
+; GFX9-NEXT: v_lshl_or_b32 v5, v14, 31, v5
+; GFX9-NEXT: v_sub_u32_e32 v10, 64, v13
+; GFX9-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[10:11], v10, v[6:7]
+; GFX9-NEXT: v_add_u32_e32 v14, 0xffffffc0, v13
+; GFX9-NEXT: v_or_b32_e32 v10, v8, v10
+; GFX9-NEXT: v_or_b32_e32 v11, v9, v11
+; GFX9-NEXT: v_lshrrev_b64 v[8:9], v13, v[6:7]
+; GFX9-NEXT: v_lshrrev_b64 v[6:7], v14, v[6:7]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v4, v16, v4
+; GFX9-NEXT: v_or_b32_e32 v5, v18, v5
+; GFX9-NEXT: v_or_b32_e32 v6, v17, v6
+; GFX9-NEXT: v_or_b32_e32 v7, v12, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fshl_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX10-NEXT: v_mov_b32_e32 v18, 0
-; GFX10-NEXT: v_not_b32_e32 v23, v16
-; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[17:18]
-; GFX10-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX10-NEXT: v_cmp_ne_u64_e64 s4, 0, v[17:18]
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v22, v1, v11, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v21, v0, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v19, v5, v15, s4
+; GFX10-NEXT: v_and_b32_e32 v19, 0x7f, v16
; GFX10-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX10-NEXT: v_cndmask_b32_e64 v13, v15, v13, s4
-; GFX10-NEXT: v_lshlrev_b64 v[10:11], v16, v[21:22]
-; GFX10-NEXT: v_cndmask_b32_e64 v12, v14, v12, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT: v_lshrrev_b64 v[2:3], 1, v[21:22]
-; GFX10-NEXT: v_lshrrev_b64 v[17:18], v23, v[8:9]
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v7, v5, s4
-; GFX10-NEXT: v_not_b32_e32 v9, v20
-; GFX10-NEXT: v_or_b32_e32 v8, v11, v18
-; GFX10-NEXT: v_cndmask_b32_e64 v18, v4, v14, s4
-; GFX10-NEXT: v_lshrrev_b64 v[11:12], 1, v[12:13]
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v4, s4
-; GFX10-NEXT: v_lshlrev_b64 v[13:14], v16, v[0:1]
-; GFX10-NEXT: v_lshrrev_b64 v[1:2], v23, v[2:3]
-; GFX10-NEXT: v_lshrrev_b64 v[6:7], 1, v[18:19]
-; GFX10-NEXT: v_lshlrev_b64 v[15:16], v20, v[18:19]
-; GFX10-NEXT: v_lshrrev_b64 v[11:12], v9, v[11:12]
-; GFX10-NEXT: v_lshlrev_b64 v[18:19], v20, v[4:5]
-; GFX10-NEXT: v_or_b32_e32 v0, v10, v17
-; GFX10-NEXT: v_or_b32_e32 v3, v14, v2
-; GFX10-NEXT: v_lshrrev_b64 v[6:7], v9, v[6:7]
-; GFX10-NEXT: v_or_b32_e32 v2, v13, v1
-; GFX10-NEXT: v_or_b32_e32 v5, v16, v12
-; GFX10-NEXT: v_or_b32_e32 v4, v15, v11
-; GFX10-NEXT: v_mov_b32_e32 v1, v8
-; GFX10-NEXT: v_or_b32_e32 v7, v19, v7
-; GFX10-NEXT: v_or_b32_e32 v6, v18, v6
+; GFX10-NEXT: v_bfi_b32 v29, v16, 0, 0x7f
+; GFX10-NEXT: v_sub_nc_u32_e32 v21, 64, v19
+; GFX10-NEXT: v_add_nc_u32_e32 v25, 0xffffffc0, v19
+; GFX10-NEXT: v_lshlrev_b64 v[23:24], v19, v[2:3]
+; GFX10-NEXT: v_lshl_or_b32 v9, v10, 31, v9
+; GFX10-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX10-NEXT: v_lshrrev_b64 v[21:22], v21, v[0:1]
+; GFX10-NEXT: v_sub_nc_u32_e32 v16, 64, v29
+; GFX10-NEXT: v_lshlrev_b64 v[17:18], v19, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v25, v[0:1]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v19
+; GFX10-NEXT: v_lshrrev_b64 v[25:26], v29, v[8:9]
+; GFX10-NEXT: v_or_b32_e32 v21, v21, v23
+; GFX10-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v29
+; GFX10-NEXT: v_lshlrev_b64 v[27:28], v16, v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e32 v30, 0, v17, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v19
+; GFX10-NEXT: v_cndmask_b32_e32 v21, v0, v21, vcc_lo
+; GFX10-NEXT: v_or_b32_e32 v0, v22, v24
+; GFX10-NEXT: v_lshrrev_b64 v[16:17], v23, v[10:11]
+; GFX10-NEXT: v_or_b32_e32 v19, v25, v27
+; GFX10-NEXT: v_cmp_gt_u32_e64 s5, 64, v29
+; GFX10-NEXT: v_or_b32_e32 v22, v26, v28
+; GFX10-NEXT: v_cndmask_b32_e32 v23, v1, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29
+; GFX10-NEXT: v_cndmask_b32_e64 v16, v16, v19, s5
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v29, v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v17, v22, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v22, v23, v3, s4
+; GFX10-NEXT: v_and_b32_e32 v23, 0x7f, v20
+; GFX10-NEXT: v_cndmask_b32_e64 v21, v21, v2, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v16, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v10, v9, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
+; GFX10-NEXT: v_sub_nc_u32_e32 v10, 64, v23
+; GFX10-NEXT: v_bfi_b32 v20, v20, 0, 0x7f
+; GFX10-NEXT: v_cndmask_b32_e64 v24, 0, v0, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v25, 0, v1, s5
+; GFX10-NEXT: v_or_b32_e32 v0, v30, v2
+; GFX10-NEXT: v_or_b32_e32 v1, v18, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v23
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], v10, v[4:5]
+; GFX10-NEXT: v_lshlrev_b64 v[10:11], v23, v[6:7]
+; GFX10-NEXT: v_lshl_or_b32 v9, v14, 31, v9
+; GFX10-NEXT: v_lshrrev_b64 v[14:15], 1, v[14:15]
+; GFX10-NEXT: v_sub_nc_u32_e32 v18, 64, v20
+; GFX10-NEXT: v_lshlrev_b64 v[12:13], v23, v[4:5]
+; GFX10-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v23
+; GFX10-NEXT: v_or_b32_e32 v10, v2, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v26, 0xffffffc0, v20
+; GFX10-NEXT: v_lshrrev_b64 v[16:17], v20, v[8:9]
+; GFX10-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
+; GFX10-NEXT: v_or_b32_e32 v2, v21, v24
+; GFX10-NEXT: v_or_b32_e32 v11, v3, v11
+; GFX10-NEXT: v_cndmask_b32_e32 v21, v4, v10, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[3:4], v26, v[14:15]
+; GFX10-NEXT: v_cmp_gt_u32_e64 s5, 64, v20
+; GFX10-NEXT: v_or_b32_e32 v10, v16, v18
+; GFX10-NEXT: v_or_b32_e32 v16, v17, v19
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v23
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v20
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v10, s5
+; GFX10-NEXT: v_lshrrev_b64 v[10:11], v20, v[14:15]
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v16, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v21, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v5, v7, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v3, v8, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v4, v9, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, v10, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, v11, s5
+; GFX10-NEXT: v_or_b32_e32 v3, v22, v25
+; GFX10-NEXT: v_or_b32_e32 v4, v12, v5
+; GFX10-NEXT: v_or_b32_e32 v5, v13, v8
+; GFX10-NEXT: v_or_b32_e32 v6, v6, v9
+; GFX10-NEXT: v_or_b32_e32 v7, v7, v10
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshl_v2i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_and_b32 v17, 64, v16
-; GFX11-NEXT: v_not_b32_e32 v23, v16
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[17:18]
-; GFX11-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX11-NEXT: v_cmp_ne_u64_e64 s0, 0, v[17:18]
-; GFX11-NEXT: v_dual_cndmask_b32 v22, v1, v11 :: v_dual_cndmask_b32 v9, v11, v9
-; GFX11-NEXT: v_dual_cndmask_b32 v8, v10, v8 :: v_dual_cndmask_b32 v21, v0, v10
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e64 v19, v5, v15, s0
+; GFX11-NEXT: v_and_b32_e32 v19, 0x7f, v16
; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
+; GFX11-NEXT: v_bfi_b32 v29, v16, 0, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[17:18], v19, v[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v19
+; GFX11-NEXT: v_lshl_or_b32 v9, v10, 31, v9
+; GFX11-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v29
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v29
+; GFX11-NEXT: v_cndmask_b32_e32 v30, 0, v17, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v21, 64, v19
+; GFX11-NEXT: v_dual_cndmask_b32 v18, 0, v18 :: v_dual_add_nc_u32 v25, 0xffffffc0, v19
+; GFX11-NEXT: v_lshlrev_b64 v[23:24], v19, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[27:28], v16, v[10:11]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshrrev_b64 v[21:22], v21, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v25, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[25:26], v29, v[8:9]
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v19
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v21, v21, v23
+; GFX11-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v29
+; GFX11-NEXT: v_or_b32_e32 v19, v25, v27
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v0, v21, vcc_lo
+; GFX11-NEXT: v_or_b32_e32 v0, v22, v24
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v23, v[10:11]
+; GFX11-NEXT: v_or_b32_e32 v22, v26, v28
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v2, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v23, v1, v0, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29
+; GFX11-NEXT: v_cndmask_b32_e64 v16, v16, v19, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v29, v[10:11]
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v17, v22, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v22, v23, v3, s0
+; GFX11-NEXT: v_and_b32_e32 v23, 0x7f, v20
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v16, v8, vcc_lo
+; GFX11-NEXT: v_bfi_b32 v20, v20, 0, 0x7f
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v9, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
+; GFX11-NEXT: v_sub_nc_u32_e32 v10, 64, v23
+; GFX11-NEXT: v_cndmask_b32_e64 v24, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v25, 0, v1, s1
+; GFX11-NEXT: v_or_b32_e32 v0, v30, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v23
+; GFX11-NEXT: v_lshl_or_b32 v9, v14, 31, v9
+; GFX11-NEXT: v_lshrrev_b64 v[14:15], 1, v[14:15]
+; GFX11-NEXT: v_add_nc_u32_e32 v26, 0xffffffc0, v20
+; GFX11-NEXT: v_or_b32_e32 v1, v18, v3
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v10, v[4:5]
+; GFX11-NEXT: v_lshlrev_b64 v[10:11], v23, v[6:7]
+; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v20
+; GFX11-NEXT: v_lshlrev_b64 v[12:13], v23, v[4:5]
+; GFX11-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v23
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v20, v[8:9]
+; GFX11-NEXT: v_or_b32_e32 v10, v2, v10
+; GFX11-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
+; GFX11-NEXT: v_or_b32_e32 v2, v21, v24
+; GFX11-NEXT: v_or_b32_e32 v11, v3, v11
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[10:11], v16, v[21:22]
-; GFX11-NEXT: v_cndmask_b32_e64 v13, v15, v13, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v12, v14, v12, s0
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[21:22]
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v7, v5, s0
-; GFX11-NEXT: v_lshrrev_b64 v[17:18], v23, v[8:9]
-; GFX11-NEXT: v_not_b32_e32 v9, v20
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v8, v11, v18
-; GFX11-NEXT: v_cndmask_b32_e64 v18, v4, v14, s0
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 1, v[12:13]
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v4, s0
-; GFX11-NEXT: v_lshlrev_b64 v[13:14], v16, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[1:2], v23, v[2:3]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], 1, v[18:19]
-; GFX11-NEXT: v_lshlrev_b64 v[15:16], v20, v[18:19]
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], v9, v[11:12]
-; GFX11-NEXT: v_lshlrev_b64 v[18:19], v20, v[4:5]
-; GFX11-NEXT: v_or_b32_e32 v0, v10, v17
-; GFX11-NEXT: v_or_b32_e32 v3, v14, v2
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v9, v[6:7]
-; GFX11-NEXT: v_or_b32_e32 v2, v13, v1
-; GFX11-NEXT: v_or_b32_e32 v5, v16, v12
-; GFX11-NEXT: v_or_b32_e32 v4, v15, v11
-; GFX11-NEXT: v_mov_b32_e32 v1, v8
-; GFX11-NEXT: v_or_b32_e32 v7, v19, v7
-; GFX11-NEXT: v_or_b32_e32 v6, v18, v6
+; GFX11-NEXT: v_dual_cndmask_b32 v12, 0, v12 :: v_dual_cndmask_b32 v21, v4, v10
+; GFX11-NEXT: v_lshrrev_b64 v[3:4], v26, v[14:15]
+; GFX11-NEXT: v_or_b32_e32 v10, v16, v18
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v20
+; GFX11-NEXT: v_or_b32_e32 v16, v17, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v23
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v20
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v10, s1
+; GFX11-NEXT: v_lshrrev_b64 v[10:11], v20, v[14:15]
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v16, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v21, v6, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v5, v7, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v3, v8, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v4, v9, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v10, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v10, 0, v11, s1
+; GFX11-NEXT: v_or_b32_e32 v3, v22, v25
+; GFX11-NEXT: v_or_b32_e32 v4, v12, v5
+; GFX11-NEXT: v_or_b32_e32 v5, v13, v8
+; GFX11-NEXT: v_or_b32_e32 v6, v6, v9
+; GFX11-NEXT: v_or_b32_e32 v7, v7, v10
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 9c285a0e7e7b4..877b50b8da8d5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -44,59 +44,152 @@ define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
;
; GFX8-LABEL: s_fshr_i7:
; GFX8: ; %bb.0:
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX8-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mul_i32 s4, s3, -7
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_add_i32 s3, s3, s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mul_i32 s3, s3, -7
+; GFX8-NEXT: s_add_i32 s2, s2, s3
+; GFX8-NEXT: s_cmp_ge_u32 s2, 7
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_add_i32 s4, s2, -7
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_cmp_ge_u32 s2, 7
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_add_i32 s4, s2, -7
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
; GFX8-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX8-NEXT: s_lshl_b32 s0, s0, 7
+; GFX8-NEXT: s_sub_i32 s3, 6, s2
+; GFX8-NEXT: s_lshl_b32 s0, s0, 1
+; GFX8-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX8-NEXT: s_lshl_b32 s0, s0, s3
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_mul_i32 s1, s2, 0x2493
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_mul_i32 s1, s1, 7
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_sub_i32 s1, s2, s1
-; GFX8-NEXT: s_lshr_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshr_i7:
; GFX9: ; %bb.0:
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-NEXT: s_mul_i32 s4, s3, -7
+; GFX9-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX9-NEXT: s_add_i32 s3, s3, s4
+; GFX9-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX9-NEXT: s_mul_i32 s3, s3, -7
+; GFX9-NEXT: s_add_i32 s2, s2, s3
+; GFX9-NEXT: s_cmp_ge_u32 s2, 7
+; GFX9-NEXT: s_cselect_b32 s3, 1, 0
+; GFX9-NEXT: s_add_i32 s4, s2, -7
+; GFX9-NEXT: s_cmp_lg_u32 s3, 0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s2
+; GFX9-NEXT: s_cmp_ge_u32 s2, 7
+; GFX9-NEXT: s_cselect_b32 s3, 1, 0
+; GFX9-NEXT: s_add_i32 s4, s2, -7
+; GFX9-NEXT: s_cmp_lg_u32 s3, 0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s2
+; GFX9-NEXT: s_sub_i32 s3, 6, s2
; GFX9-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX9-NEXT: s_lshl_b32 s0, s0, 7
+; GFX9-NEXT: s_lshl_b32 s0, s0, 1
+; GFX9-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX9-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s3
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
-; GFX9-NEXT: s_mul_i32 s1, s2, 0x2493
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: s_mul_i32 s1, s1, 7
-; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX9-NEXT: s_sub_i32 s1, s2, s1
-; GFX9-NEXT: s_lshr_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshr_i7:
; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
; GFX10-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-NEXT: s_mul_i32 s4, s3, -7
+; GFX10-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX10-NEXT: s_add_i32 s3, s3, s4
+; GFX10-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX10-NEXT: s_mul_i32 s3, s3, -7
+; GFX10-NEXT: s_add_i32 s2, s2, s3
+; GFX10-NEXT: s_cmp_ge_u32 s2, 7
+; GFX10-NEXT: s_cselect_b32 s3, 1, 0
+; GFX10-NEXT: s_add_i32 s4, s2, -7
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10-NEXT: s_cselect_b32 s2, s4, s2
+; GFX10-NEXT: s_cmp_ge_u32 s2, 7
+; GFX10-NEXT: s_cselect_b32 s3, 1, 0
+; GFX10-NEXT: s_add_i32 s4, s2, -7
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10-NEXT: s_cselect_b32 s2, s4, s2
; GFX10-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX10-NEXT: s_mul_i32 s3, s2, 0x2493
-; GFX10-NEXT: s_lshl_b32 s0, s0, 7
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
+; GFX10-NEXT: s_sub_i32 s3, 6, s2
+; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX10-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshl_b32 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
-; GFX10-NEXT: s_mul_i32 s3, s3, 7
-; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX10-NEXT: s_sub_i32 s1, s2, s3
-; GFX10-NEXT: s_lshr_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i7:
; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, -7
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_i32 s3, s3, -7
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX11-NEXT: s_mul_i32 s3, s2, 0x2493
-; GFX11-NEXT: s_lshl_b32 s0, s0, 7
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_mul_i32 s3, s3, 7
-; GFX11-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX11-NEXT: s_sub_i32 s1, s2, s3
+; GFX11-NEXT: s_sub_i32 s3, 6, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s0, s0, s1
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i7 @llvm.fshr.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
@@ -233,43 +326,55 @@ define amdgpu_ps i8 @s_fshr_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
;
; GFX8-LABEL: s_fshr_i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX8-NEXT: v_perm_b32 v0, s1, v0, v1
-; GFX8-NEXT: s_and_b32 s0, s2, 7
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_xor_b32 s3, s2, -1
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 1
+; GFX8-NEXT: s_and_b32 s3, s3, 7
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s3
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshr_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s1, v0, v1
-; GFX9-NEXT: s_and_b32 s0, s2, 7
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_xor_b32 s3, s2, -1
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s0, s0, 1
+; GFX9-NEXT: s_and_b32 s3, s3, 7
+; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s3
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshr_i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX10-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX10-NEXT: s_and_b32 s0, s2, 7
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_xor_b32 s3, s2, -1
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_and_b32 s3, s3, 7
+; GFX10-NEXT: s_and_b32 s2, s2, 7
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshl_b32 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i8:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX11-NEXT: s_and_b32 s0, s2, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_xor_b32 s3, s2, -1
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s3, s3, 7
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
@@ -341,18 +446,47 @@ define i8 @v_fshr_i8(i8 %lhs, i8 %rhs, i8 %amt) {
}
define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
-; GCN-LABEL: s_fshr_i8_4:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b32 s0, s0, 4
-; GCN-NEXT: s_bfe_u32 s1, s1, 0x40004
-; GCN-NEXT: s_or_b32 s0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i8_4:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b32 s0, s0, 4
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x40004
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i8_4:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshr_b32 s1, s1, 4
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i8_4:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, 4
+; GFX9-NEXT: s_lshr_b32 s1, s1, 4
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i8_4:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshr_b32 s1, s1, 4
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i8_4:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x40004
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 4)
@@ -419,18 +553,47 @@ define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
}
define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
-; GCN-LABEL: s_fshr_i8_5:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b32 s0, s0, 3
-; GCN-NEXT: s_bfe_u32 s1, s1, 0x30005
-; GCN-NEXT: s_or_b32 s0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i8_5:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b32 s0, s0, 3
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x30005
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i8_5:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 3
+; GFX8-NEXT: s_lshr_b32 s1, s1, 5
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i8_5:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, 3
+; GFX9-NEXT: s_lshr_b32 s1, s1, 5
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i8_5:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s0, s0, 3
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshr_b32 s1, s1, 5
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i8_5:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
; GFX11-NEXT: s_lshl_b32 s0, s0, 3
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x30005
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 5
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 5)
@@ -523,69 +686,119 @@ define amdgpu_ps i16 @s_fshr_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
;
; GFX8-LABEL: s_fshr_v2i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0xc0c0105
-; GFX8-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s0, v1
-; GFX8-NEXT: v_perm_b32 v0, s1, v0, v2
-; GFX8-NEXT: s_and_b32 s0, s2, 7
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_lshr_b32 s4, s1, 8
+; GFX8-NEXT: s_xor_b32 s6, s2, -1
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_lshr_b32 s3, s0, 8
+; GFX8-NEXT: s_lshr_b32 s5, s2, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, 1
+; GFX8-NEXT: s_and_b32 s6, s6, 7
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s6
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
+; GFX8-NEXT: s_xor_b32 s2, s5, -1
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s3, 1
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_and_b32 s3, s4, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, s5, 7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshr_b32 s2, s3, s2
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshr_v2i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, s0, v1
-; GFX9-NEXT: v_perm_b32 v0, s1, v0, v2
-; GFX9-NEXT: s_and_b32 s0, s2, 7
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s4, s1, 8
+; GFX9-NEXT: s_xor_b32 s6, s2, -1
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_lshr_b32 s3, s0, 8
+; GFX9-NEXT: s_lshr_b32 s5, s2, 8
+; GFX9-NEXT: s_lshl_b32 s0, s0, 1
+; GFX9-NEXT: s_and_b32 s6, s6, 7
+; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s6
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
+; GFX9-NEXT: s_xor_b32 s2, s5, -1
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s1, s3, 1
+; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_and_b32 s3, s4, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s2, s5, 7
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX9-NEXT: s_lshr_b32 s2, s3, s2
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshr_v2i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX10-NEXT: s_bfe_u32 s3, s2, 0x30008
-; GFX10-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX10-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX10-NEXT: s_and_b32 s0, s2, 7
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s3, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s0, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_xor_b32 s5, s2, -1
+; GFX10-NEXT: s_lshr_b32 s3, s0, 8
+; GFX10-NEXT: s_lshr_b32 s4, s1, 8
+; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_and_b32 s5, s5, 7
+; GFX10-NEXT: s_lshr_b32 s6, s2, 8
+; GFX10-NEXT: s_lshl_b32 s0, s0, s5
+; GFX10-NEXT: s_xor_b32 s5, s6, -1
+; GFX10-NEXT: s_and_b32 s4, s4, 0xff
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s3, s3, 1
+; GFX10-NEXT: s_and_b32 s5, s5, 7
+; GFX10-NEXT: s_and_b32 s6, s6, 7
+; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX10-NEXT: s_and_b32 s2, s2, 7
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshl_b32 s3, s3, s5
+; GFX10-NEXT: s_lshr_b32 s4, s4, s6
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
+; GFX10-NEXT: s_or_b32 s2, s3, s4
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: s_and_b32 s1, s2, 0xff
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v2i8:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX11-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX11-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX11-NEXT: s_and_b32 s1, s2, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_xor_b32 s5, s2, -1
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s5, s5, 7
+; GFX11-NEXT: s_lshr_b32 s6, s2, 8
+; GFX11-NEXT: s_lshl_b32 s0, s0, s5
+; GFX11-NEXT: s_xor_b32 s5, s6, -1
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_and_b32 s5, s5, 7
+; GFX11-NEXT: s_and_b32 s6, s6, 7
+; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s3, s3, s5
+; GFX11-NEXT: s_lshr_b32 s4, s4, s6
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_or_b32 s2, s3, s4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_and_b32 s1, s2, 0xff
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -2624,45 +2837,45 @@ define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
;
; GFX8-LABEL: s_fshr_i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: s_and_b32 s3, s2, 15
; GFX8-NEXT: s_andn2_b32 s2, 15, s2
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: s_lshr_b32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: s_lshl_b32 s0, s0, s2
+; GFX8-NEXT: s_lshr_b32 s1, s1, s3
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshr_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
; GFX9-NEXT: s_and_b32 s3, s2, 15
; GFX9-NEXT: s_andn2_b32 s2, 15, s2
; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: s_lshr_b32 s1, s1, s3
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
; GFX9-NEXT: s_lshl_b32 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s1, s1, s3
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshr_i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
; GFX10-NEXT: s_and_b32 s3, s2, 15
; GFX10-NEXT: s_andn2_b32 s2, 15, s2
; GFX10-NEXT: s_lshl_b32 s0, s0, 1
-; GFX10-NEXT: s_lshr_b32 s1, s1, s3
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
; GFX10-NEXT: s_lshl_b32 s0, s0, s2
+; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
; GFX11-NEXT: s_and_b32 s3, s2, 15
; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
@@ -2681,16 +2894,16 @@ define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-LABEL: s_fshr_i16_4:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 4
; GFX8-NEXT: s_lshl_b32 s0, s0, 12
+; GFX8-NEXT: s_lshr_b32 s1, s1, 4
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshr_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_lshl_b32 s0, s0, 12
+; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -2725,16 +2938,16 @@ define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-LABEL: s_fshr_i16_5:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 5
; GFX8-NEXT: s_lshl_b32 s0, s0, 11
+; GFX8-NEXT: s_lshr_b32 s1, s1, 5
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_fshr_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 5
; GFX9-NEXT: s_lshl_b32 s0, s0, 11
+; GFX9-NEXT: s_lshr_b32 s1, s1, 5
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -2986,7 +3199,10 @@ define amdgpu_ps half @v_fshr_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX11-TRUE16-LABEL: v_fshr_i16_ssv:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v0.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, s0
@@ -3120,8 +3336,8 @@ define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
@@ -3189,40 +3405,69 @@ define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
;
; GFX9-LABEL: s_fshr_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, 1, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_xor_b32 s0, s2, -1
-; GFX9-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, s0, v0
-; GFX9-NEXT: s_and_b32 s0, s2, 0xf000f
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, s0, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT: s_lshl_b32 s4, s4, 1
+; GFX9-NEXT: s_and_b32 s3, s2, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s2, 0xf000f, s2
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s2, s4, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshr_b32 s2, s2, s3
+; GFX9-NEXT: s_lshr_b32 s1, s1, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshr_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 1, s0 op_sel_hi:[0,1]
-; GFX10-NEXT: s_xor_b32 s0, s2, -1
-; GFX10-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX10-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX10-NEXT: v_pk_lshrrev_b16 v1, s2, s1
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, s0, v0
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT: s_lshl_b32 s3, s3, 1
+; GFX10-NEXT: s_and_b32 s4, s2, 0xf000f
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT: s_andn2_b32 s2, 0xf000f, s2
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s2, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, s5
+; GFX10-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10-NEXT: s_lshr_b32 s5, s4, 16
+; GFX10-NEXT: s_lshr_b32 s3, s3, s4
+; GFX10-NEXT: s_lshr_b32 s1, s1, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v2i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_xor_b32 s0, s2, -1
-; GFX11-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX11-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, s2, s1
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_and_b32 s4, s2, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s2, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s2, s3, s5
+; GFX11-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_lshr_b32 s5, s4, 16
+; GFX11-NEXT: s_lshr_b32 s3, s3, s4
+; GFX11-NEXT: s_lshr_b32 s1, s1, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to i32
@@ -3399,36 +3644,44 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
;
; GFX9-LABEL: v_fshr_v2i16_ssv:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, 1, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_and_b32_e32 v2, 0xf000f, v2
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: v_and_b32_e32 v1, 0xf000f, v0
+; GFX9-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT: s_lshl_b32 s2, s2, 1
; GFX9-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, v2, v1
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, v0, s1
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, v0, s0
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, v1, s1
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_v2i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 1, s0 op_sel_hi:[0,1]
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
; GFX10-NEXT: v_and_b32_e32 v0, 0xf000f, v0
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT: s_lshl_b32 s2, s2, 1
; GFX10-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX10-NEXT: v_pk_lshrrev_b16 v0, v0, s1
-; GFX10-NEXT: v_pk_lshlrev_b16 v1, v1, v2
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, v1, s0
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_v2i16_ssv:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX11-NEXT: v_pk_lshlrev_b16 v2, 1, s0 op_sel_hi:[0,1]
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s2, s2, 1
; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX11-NEXT: v_pk_lshrrev_b16 v0, v0, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v1, v2
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v1, s0
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
@@ -3477,36 +3730,55 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
;
; GFX9-LABEL: v_fshr_v2i16_svs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, 1, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_xor_b32 s0, s1, -1
-; GFX9-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, s0, v1
-; GFX9-NEXT: s_and_b32 s0, s1, 0xf000f
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, s0, v0
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT: s_lshl_b32 s3, s3, 1
+; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s1, 0xf000f, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s1, s3, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_pk_lshrrev_b16 v0, s2, v0
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_v2i16_svs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_lshlrev_b16 v1, 1, s0 op_sel_hi:[0,1]
-; GFX10-NEXT: s_xor_b32 s0, s1, -1
-; GFX10-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX10-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX10-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX10-NEXT: v_pk_lshlrev_b16 v1, s0, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT: s_lshl_b32 s2, s2, 1
+; GFX10-NEXT: s_and_b32 s3, s1, 0xf000f
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_andn2_b32 s1, 0xf000f, s1
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshr_b32 s4, s1, 16
+; GFX10-NEXT: v_pk_lshrrev_b16 v0, s3, v0
+; GFX10-NEXT: s_lshl_b32 s0, s0, s1
+; GFX10-NEXT: s_lshl_b32 s1, s2, s4
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_v2i16_svs:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_xor_b32 s0, s1, -1
-; GFX11-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX11-NEXT: s_and_b32 s0, s0, 0xf000f
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, s0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s2, s2, 1
+; GFX11-NEXT: s_and_b32 s3, s1, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, s3, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
@@ -3555,37 +3827,49 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
;
; GFX9-LABEL: v_fshr_v2i16_vss:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s2, s1, -1
+; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s1, 0xf000f, s1
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX9-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, s2, v0
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, s1, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, s1, v0
+; GFX9-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
+; GFX9-NEXT: s_lshr_b32 s0, s0, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_v2i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: s_xor_b32 s2, s1, -1
-; GFX10-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX10-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX10-NEXT: v_pk_lshrrev_b16 v1, s1, s0
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, s2, v0
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX10-NEXT: s_andn2_b32 s1, 0xf000f, s1
+; GFX10-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX10-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10-NEXT: s_lshr_b32 s4, s2, 16
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, s1, v0
+; GFX10-NEXT: s_lshr_b32 s1, s3, s2
+; GFX10-NEXT: s_lshr_b32 s0, s0, s4
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_v2i16_vss:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_xor_b32 s2, s1, -1
-; GFX11-NEXT: s_and_b32 s1, s1, 0xf000f
-; GFX11-NEXT: s_and_b32 s2, s2, 0xf000f
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, s1, s0
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, s2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, s1, v0
+; GFX11-NEXT: s_lshr_b32 s1, s3, s2
+; GFX11-NEXT: s_lshr_b32 s0, s0, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
@@ -3742,43 +4026,83 @@ define amdgpu_ps i48 @s_fshr_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s5, 15
-; GFX11-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s5
-; GFX11-NEXT: s_lshl_b32 s9, s1, 1
-; GFX11-NEXT: s_lshr_b32 s6, s7, s6
-; GFX11-NEXT: s_lshl_b32 s7, s9, s8
-; GFX11-NEXT: s_lshr_b32 s1, s1, 15
-; GFX11-NEXT: s_or_b32 s6, s7, s6
-; GFX11-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-NEXT: s_and_b32 s1, s1, 0x1fffe
-; GFX11-NEXT: s_and_not1_b32 s7, 15, s7
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX11-NEXT: s_lshl_b32 s1, s1, s7
-; GFX11-NEXT: s_lshr_b32 s3, s3, s5
-; GFX11-NEXT: s_and_b32 s5, s4, 15
-; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s4
-; GFX11-NEXT: s_lshl_b32 s9, s0, 1
-; GFX11-NEXT: s_lshr_b32 s5, s7, s5
-; GFX11-NEXT: s_lshl_b32 s7, s9, s8
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshr_b32 s0, s0, 15
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s8
-; GFX11-NEXT: s_and_b32 s0, s0, 0x1fffe
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX11-NEXT: s_lshl_b32 s0, s0, s8
-; GFX11-NEXT: s_lshr_b32 s2, s2, s4
-; GFX11-NEXT: s_or_b32 s4, s7, s5
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_fshr_v3i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s6, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s7, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s6, s7
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s5
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s5, 0xf000f
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s5, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s6, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s5, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_fshr_v3i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s5, 15
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s3, 0xffff
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s5
+; GFX11-FAKE16-NEXT: s_lshl_b32 s9, s1, 1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s7, s6
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s9, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 15
+; GFX11-FAKE16-NEXT: s_or_b32 s6, s7, s6
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0x1fffe
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s7, 15, s7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT: s_bfe_u32 s5, s5, 0x40010
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s5
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s4, 15
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s4
+; GFX11-FAKE16-NEXT: s_lshl_b32 s9, s0, 1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s7, s5
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s9, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 15
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x1fffe
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s7, s5
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
%cast = bitcast <3 x i16> %result to i48
ret i48 %cast
@@ -4004,116 +4328,122 @@ define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
;
; GFX9-LABEL: s_fshr_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s6, s5, 15
-; GFX9-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX9-NEXT: s_lshr_b32 s6, s7, s6
-; GFX9-NEXT: s_andn2_b32 s7, 15, s5
-; GFX9-NEXT: s_lshl_b32 s8, s1, 1
-; GFX9-NEXT: s_lshl_b32 s7, s8, s7
-; GFX9-NEXT: s_or_b32 s6, s7, s6
-; GFX9-NEXT: s_lshr_b32 s7, s5, 16
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x100010
-; GFX9-NEXT: s_andn2_b32 s7, 15, s7
-; GFX9-NEXT: s_lshl_b32 s1, s1, 1
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX9-NEXT: s_lshl_b32 s1, s1, s7
-; GFX9-NEXT: s_lshr_b32 s3, s3, s5
-; GFX9-NEXT: s_or_b32 s1, s1, s3
-; GFX9-NEXT: s_and_b32 s3, s4, 15
-; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX9-NEXT: s_lshr_b32 s3, s5, s3
-; GFX9-NEXT: s_andn2_b32 s5, 15, s4
-; GFX9-NEXT: s_lshl_b32 s6, s0, 1
-; GFX9-NEXT: s_lshl_b32 s5, s6, s5
-; GFX9-NEXT: s_or_b32 s3, s5, s3
-; GFX9-NEXT: s_lshr_b32 s5, s4, 16
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX9-NEXT: s_andn2_b32 s5, 15, s5
-; GFX9-NEXT: s_lshl_b32 s0, s0, 1
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT: s_lshl_b32 s7, s7, 1
+; GFX9-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s7
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_lshr_b32 s8, s4, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s4
+; GFX9-NEXT: s_lshl_b32 s4, s7, s8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX9-NEXT: s_lshl_b32 s0, s0, s5
-; GFX9-NEXT: s_lshr_b32 s2, s2, s4
+; GFX9-NEXT: s_lshr_b32 s7, s6, 16
+; GFX9-NEXT: s_lshr_b32 s4, s4, s6
+; GFX9-NEXT: s_lshr_b32 s2, s2, s7
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s2
; GFX9-NEXT: s_or_b32 s0, s0, s2
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX9-NEXT: s_and_b32 s2, s5, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s5
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX9-NEXT: s_lshl_b32 s5, s5, 1
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
+; GFX9-NEXT: s_lshr_b32 s6, s4, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, s4
+; GFX9-NEXT: s_lshl_b32 s4, s5, s6
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_and_b32 s4, s3, 0xffff
+; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-NEXT: s_lshr_b32 s2, s4, s2
+; GFX9-NEXT: s_lshr_b32 s3, s3, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT: s_or_b32 s1, s1, s2
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fshr_v4i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s6, s5, 15
-; GFX10-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX10-NEXT: s_andn2_b32 s8, 15, s5
-; GFX10-NEXT: s_lshl_b32 s9, s1, 1
-; GFX10-NEXT: s_lshr_b32 s6, s7, s6
-; GFX10-NEXT: s_lshl_b32 s7, s9, s8
-; GFX10-NEXT: s_bfe_u32 s1, s1, 0x100010
-; GFX10-NEXT: s_or_b32 s6, s7, s6
-; GFX10-NEXT: s_lshr_b32 s7, s5, 16
-; GFX10-NEXT: s_lshl_b32 s1, s1, 1
-; GFX10-NEXT: s_andn2_b32 s7, 15, s7
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
-; GFX10-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX10-NEXT: s_lshl_b32 s1, s1, s7
-; GFX10-NEXT: s_lshr_b32 s3, s3, s5
-; GFX10-NEXT: s_and_b32 s5, s4, 15
-; GFX10-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX10-NEXT: s_andn2_b32 s8, 15, s4
-; GFX10-NEXT: s_lshl_b32 s9, s0, 1
-; GFX10-NEXT: s_lshr_b32 s5, s7, s5
-; GFX10-NEXT: s_lshl_b32 s7, s9, s8
+; GFX10-NEXT: s_lshr_b32 s6, s0, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT: s_lshl_b32 s6, s6, 1
+; GFX10-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX10-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX10-NEXT: s_lshr_b32 s6, s0, 16
; GFX10-NEXT: s_lshr_b32 s8, s4, 16
-; GFX10-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX10-NEXT: s_andn2_b32 s8, 15, s8
-; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, s4
+; GFX10-NEXT: s_lshl_b32 s4, s6, s8
+; GFX10-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX10-NEXT: s_lshr_b32 s4, s1, 16
; GFX10-NEXT: s_lshr_b32 s2, s2, 16
-; GFX10-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX10-NEXT: s_lshl_b32 s0, s0, s8
-; GFX10-NEXT: s_lshr_b32 s2, s2, s4
-; GFX10-NEXT: s_or_b32 s4, s7, s5
+; GFX10-NEXT: s_lshr_b32 s8, s7, 16
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX10-NEXT: s_lshl_b32 s4, s4, 1
+; GFX10-NEXT: s_lshr_b32 s6, s6, s7
+; GFX10-NEXT: s_lshr_b32 s2, s2, s8
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT: s_andn2_b32 s4, 0xf000f, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX10-NEXT: s_and_b32 s6, s5, 0xf000f
+; GFX10-NEXT: s_lshr_b32 s5, s1, 16
+; GFX10-NEXT: s_lshr_b32 s7, s4, 16
+; GFX10-NEXT: s_lshl_b32 s1, s1, s4
+; GFX10-NEXT: s_lshl_b32 s4, s5, s7
+; GFX10-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX10-NEXT: s_lshr_b32 s3, s3, 16
+; GFX10-NEXT: s_lshr_b32 s7, s6, 16
+; GFX10-NEXT: s_lshr_b32 s5, s5, s6
+; GFX10-NEXT: s_lshr_b32 s3, s3, s7
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT: s_pack_ll_b32_b16 s3, s5, s3
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v4i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s5, 15
-; GFX11-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s5
-; GFX11-NEXT: s_lshl_b32 s9, s1, 1
-; GFX11-NEXT: s_lshr_b32 s6, s7, s6
-; GFX11-NEXT: s_lshl_b32 s7, s9, s8
-; GFX11-NEXT: s_lshr_b32 s1, s1, 15
-; GFX11-NEXT: s_or_b32 s6, s7, s6
-; GFX11-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-NEXT: s_and_b32 s1, s1, 0x1fffe
-; GFX11-NEXT: s_and_not1_b32 s7, 15, s7
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX11-NEXT: s_lshl_b32 s1, s1, s7
-; GFX11-NEXT: s_lshr_b32 s3, s3, s5
-; GFX11-NEXT: s_and_b32 s5, s4, 15
-; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s4
-; GFX11-NEXT: s_lshl_b32 s9, s0, 1
-; GFX11-NEXT: s_lshr_b32 s5, s7, s5
-; GFX11-NEXT: s_lshl_b32 s7, s9, s8
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s6, s6, 1
+; GFX11-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshr_b32 s0, s0, 15
-; GFX11-NEXT: s_and_not1_b32 s8, 15, s8
-; GFX11-NEXT: s_and_b32 s0, s0, 0x1fffe
+; GFX11-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s8
+; GFX11-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX11-NEXT: s_lshl_b32 s0, s0, s8
-; GFX11-NEXT: s_lshr_b32 s2, s2, s4
-; GFX11-NEXT: s_or_b32 s4, s7, s5
+; GFX11-NEXT: s_lshr_b32 s8, s7, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX11-NEXT: s_lshl_b32 s4, s4, 1
+; GFX11-NEXT: s_lshr_b32 s6, s6, s7
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX11-NEXT: s_and_b32 s6, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s5, s7
+; GFX11-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s6, 16
+; GFX11-NEXT: s_lshr_b32 s5, s5, s6
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
; GFX11-NEXT: s_or_b32 s0, s0, s2
; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
; GFX11-NEXT: ; return to shader part epilog
%result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <2 x i32>
@@ -4424,19 +4754,19 @@ define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
; GFX6-LABEL: v_fshr_i64_5:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshr_b64 v[4:5], v[2:3], 5
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 27, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v3, v2, 5
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[2:3], 5
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 27, v4
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_i64_5:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b64 v[4:5], 5, v[2:3]
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 27, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v0, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v3, v2, 5
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 27, v4
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshr_i64_5:
@@ -4539,54 +4869,60 @@ define i64 @v_fshr_i64_48(i64 %lhs, i64 %rhs) {
define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64 %amt) {
; GFX6-LABEL: v_fshr_i64_ssv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshr_b64 v[2:3], s[2:3], v0
; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX6-NEXT: v_not_b32_e32 v0, v0
-; GFX6-NEXT: v_lshl_b64 v[0:1], s[0:1], v0
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_bfi_b32 v1, v0, 0, 63
+; GFX6-NEXT: v_and_b32_e32 v0, 63, v0
+; GFX6-NEXT: v_lshl_b64 v[1:2], s[0:1], v1
+; GFX6-NEXT: v_lshr_b64 v[3:4], s[2:3], v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v3
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v4
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i64_ssv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v0, s[2:3]
; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: v_not_b32_e32 v0, v0
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v0, s[0:1]
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_bfi_b32 v1, v0, 0, 63
+; GFX8-NEXT: v_and_b32_e32 v0, 63, v0
+; GFX8-NEXT: v_lshlrev_b64 v[1:2], v1, s[0:1]
+; GFX8-NEXT: v_lshrrev_b64 v[3:4], v0, s[2:3]
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshr_i64_ssv:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], v0, s[2:3]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX9-NEXT: v_not_b32_e32 v0, v0
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, s[0:1]
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfi_b32 v1, v0, 0, 63
+; GFX9-NEXT: v_and_b32_e32 v0, 63, v0
+; GFX9-NEXT: v_lshlrev_b64 v[1:2], v1, s[0:1]
+; GFX9-NEXT: v_lshrrev_b64 v[3:4], v0, s[2:3]
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v2, v4
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_i64_ssv:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_not_b32_e32 v2, v0
+; GFX10-NEXT: v_bfi_b32 v1, v0, 0, 63
+; GFX10-NEXT: v_and_b32_e32 v2, 63, v0
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], v0, s[2:3]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v2, s[0:1]
-; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_i64_ssv:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_not_b32_e32 v2, v0
+; GFX11-NEXT: v_bfi_b32 v1, v0, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v2, 63, v0
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v0, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v2, s[0:1]
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -4596,55 +4932,67 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg %amt) {
; GFX6-LABEL: v_fshr_i64_svs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s2
; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX6-NEXT: s_not_b32 s2, s2
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: s_not_b32 s3, s2
+; GFX6-NEXT: s_and_b32 s2, s2, 63
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i64_svs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: s_not_b32 s2, s2
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: s_not_b32 s3, s2
+; GFX8-NEXT: s_and_b32 s2, s2, 63
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshr_i64_svs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX9-NEXT: s_not_b32 s2, s2
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: s_not_b32 s3, s2
+; GFX9-NEXT: s_and_b32 s2, s2, 63
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_i64_svs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_not_b32 s2, s2
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT: s_not_b32 s3, s2
+; GFX10-NEXT: s_and_b32 s2, s2, 63
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_i64_svs:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s2, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 63
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -4655,53 +5003,62 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX6-LABEL: v_fshr_i64_vss:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; GFX6-NEXT: s_not_b32 s3, s2
+; GFX6-NEXT: s_andn2_b32 s3, 63, s2
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], s3
; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i64_vss:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX8-NEXT: s_not_b32 s3, s2
+; GFX8-NEXT: s_andn2_b32 s3, 63, s2
; GFX8-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshr_i64_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX9-NEXT: s_not_b32 s3, s2
+; GFX9-NEXT: s_andn2_b32 s3, 63, s2
; GFX9-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_i64_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX10-NEXT: s_not_b32 s3, s2
+; GFX10-NEXT: s_andn2_b32 s3, 63, s2
; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_i64_vss:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -5666,31 +6023,23 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
; GFX6-LABEL: v_fshr_i128_65:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshr_b64 v[3:4], v[6:7], 1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 31, v0
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[0:1], 1
-; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 31, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v7, v6, 1
-; GFX6-NEXT: v_or_b32_e32 v3, v2, v9
-; GFX6-NEXT: v_alignbit_b32 v2, v1, v0, 1
-; GFX6-NEXT: v_mov_b32_e32 v0, v5
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 31, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 31, v2
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[0:1], 1
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[6:7], 1
+; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_i128_65:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b64 v[3:4], 1, v[6:7]
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 31, v0
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], 1, v[0:1]
-; GFX8-NEXT: v_or_b32_e32 v4, v3, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 31, v2
-; GFX8-NEXT: v_alignbit_b32 v5, v7, v6, 1
-; GFX8-NEXT: v_or_b32_e32 v3, v2, v9
-; GFX8-NEXT: v_alignbit_b32 v2, v1, v0, 1
-; GFX8-NEXT: v_mov_b32_e32 v0, v5
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 31, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 31, v2
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[0:1]
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[6:7]
+; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fshr_i128_65:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
index 6d118279f2117..8d76171fa395d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
@@ -109,10 +109,23 @@ define amdgpu_ps void @fsub_s64_div(double %a, double %b, ptr addrspace(1) %ptr)
}
define amdgpu_ps <2 x half> @fsub_v2s16_uniform(<2 x half> inreg %a, <2 x half> inreg %b) {
-; GCN-LABEL: fsub_v2s16_uniform:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_pk_add_f16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
-; GCN-NEXT: ; return to shader part epilog
+; GFX11-LABEL: fsub_v2s16_uniform:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_pk_add_f16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2s16_uniform:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshr_b32 s2, s1, 16
+; GFX12-NEXT: s_xor_b32 s1, s1, 0x8000
+; GFX12-NEXT: s_xor_b32 s2, s2, 0x8000
+; GFX12-NEXT: s_lshr_b32 s3, s0, 16
+; GFX12-NEXT: s_add_f16 s0, s0, s1
+; GFX12-NEXT: s_add_f16 s1, s3, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
%fsub = fsub <2 x half> %a, %b
ret <2 x half> %fsub
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/icmp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/icmp.ll
index 3c05f9b61ca21..6073c86aee59f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/icmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/icmp.ll
@@ -74,8 +74,8 @@ define void @icmp_i16_uniform(i16 inreg %a, i16 inreg %b, ptr addrspace(1) %p) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bfe_u32 s2, s0, 0x100000
-; GFX12-NEXT: s_bfe_u32 s3, s1, 0x100000
+; GFX12-NEXT: s_and_b32 s2, 0xffff, s0
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s1
; GFX12-NEXT: s_sext_i32_i16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_cmp_eq_u32 s2, s3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
index de1608e8b6e07..7e9d37df9979d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
@@ -85,20 +85,22 @@ define amdgpu_ps void @insertelement_s_v2i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX11-TRUE16-LABEL: insertelement_s_v2i8_s_s:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s4, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, 0xff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v2, v0, s[2:3]
+; GFX11-TRUE16-NEXT: s_mov_b32 m0, s5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: global_load_u16 v0, v0, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v0, s0, v2
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s5, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, 0xff, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_and_b32 v3, s0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v2, v3, v2
-; GFX11-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x100000
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_movreld_b32 s0, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: insertelement_s_v2i8_s_s:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index eeac37bd0abb3..4827e54839722 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -1415,22 +1415,38 @@ entry:
define amdgpu_ps void @dyn_insertelement_v8f64_s_v_s(<8 x double> inreg %vec, double %val, i32 inreg %idx) {
; GPRIDX-LABEL: dyn_insertelement_v8f64_s_v_s:
; GPRIDX: ; %bb.0: ; %entry
-; GPRIDX-NEXT: v_mov_b32_e32 v2, s2
-; GPRIDX-NEXT: v_mov_b32_e32 v3, s3
-; GPRIDX-NEXT: v_mov_b32_e32 v4, s4
-; GPRIDX-NEXT: v_mov_b32_e32 v5, s5
-; GPRIDX-NEXT: v_mov_b32_e32 v6, s6
-; GPRIDX-NEXT: v_mov_b32_e32 v7, s7
-; GPRIDX-NEXT: v_mov_b32_e32 v8, s8
-; GPRIDX-NEXT: v_mov_b32_e32 v9, s9
-; GPRIDX-NEXT: v_mov_b32_e32 v10, s10
-; GPRIDX-NEXT: v_mov_b32_e32 v11, s11
-; GPRIDX-NEXT: v_mov_b32_e32 v12, s12
-; GPRIDX-NEXT: v_mov_b32_e32 v13, s13
-; GPRIDX-NEXT: v_mov_b32_e32 v14, s14
-; GPRIDX-NEXT: v_mov_b32_e32 v15, s15
-; GPRIDX-NEXT: v_mov_b32_e32 v16, s16
-; GPRIDX-NEXT: v_mov_b32_e32 v17, s17
+; GPRIDX-NEXT: s_mov_b32 s0, s2
+; GPRIDX-NEXT: s_mov_b32 s1, s3
+; GPRIDX-NEXT: s_mov_b32 s2, s4
+; GPRIDX-NEXT: s_mov_b32 s3, s5
+; GPRIDX-NEXT: s_mov_b32 s4, s6
+; GPRIDX-NEXT: s_mov_b32 s5, s7
+; GPRIDX-NEXT: s_mov_b32 s6, s8
+; GPRIDX-NEXT: s_mov_b32 s7, s9
+; GPRIDX-NEXT: s_mov_b32 s8, s10
+; GPRIDX-NEXT: s_mov_b32 s9, s11
+; GPRIDX-NEXT: s_mov_b32 s10, s12
+; GPRIDX-NEXT: s_mov_b32 s11, s13
+; GPRIDX-NEXT: s_mov_b32 s12, s14
+; GPRIDX-NEXT: s_mov_b32 s13, s15
+; GPRIDX-NEXT: s_mov_b32 s14, s16
+; GPRIDX-NEXT: s_mov_b32 s15, s17
+; GPRIDX-NEXT: v_mov_b32_e32 v17, s15
+; GPRIDX-NEXT: v_mov_b32_e32 v16, s14
+; GPRIDX-NEXT: v_mov_b32_e32 v15, s13
+; GPRIDX-NEXT: v_mov_b32_e32 v14, s12
+; GPRIDX-NEXT: v_mov_b32_e32 v13, s11
+; GPRIDX-NEXT: v_mov_b32_e32 v12, s10
+; GPRIDX-NEXT: v_mov_b32_e32 v11, s9
+; GPRIDX-NEXT: v_mov_b32_e32 v10, s8
+; GPRIDX-NEXT: v_mov_b32_e32 v9, s7
+; GPRIDX-NEXT: v_mov_b32_e32 v8, s6
+; GPRIDX-NEXT: v_mov_b32_e32 v7, s5
+; GPRIDX-NEXT: v_mov_b32_e32 v6, s4
+; GPRIDX-NEXT: v_mov_b32_e32 v5, s3
+; GPRIDX-NEXT: v_mov_b32_e32 v4, s2
+; GPRIDX-NEXT: v_mov_b32_e32 v3, s1
+; GPRIDX-NEXT: v_mov_b32_e32 v2, s0
; GPRIDX-NEXT: s_lshl_b32 s0, s18, 1
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
; GPRIDX-NEXT: v_mov_b32_e32 v2, v0
@@ -1448,23 +1464,39 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_s(<8 x double> inreg %vec, do
;
; GFX10-LABEL: dyn_insertelement_v8f64_s_v_s:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: s_mov_b32 s8, s10
+; GFX10-NEXT: s_mov_b32 s9, s11
+; GFX10-NEXT: s_mov_b32 s10, s12
+; GFX10-NEXT: s_mov_b32 s11, s13
+; GFX10-NEXT: s_mov_b32 s12, s14
+; GFX10-NEXT: s_mov_b32 s13, s15
+; GFX10-NEXT: s_mov_b32 s14, s16
+; GFX10-NEXT: s_mov_b32 s15, s17
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_lshl_b32 m0, s18, 1
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: v_mov_b32_e32 v8, s8
-; GFX10-NEXT: v_mov_b32_e32 v9, s9
-; GFX10-NEXT: v_mov_b32_e32 v10, s10
-; GFX10-NEXT: v_mov_b32_e32 v11, s11
-; GFX10-NEXT: v_mov_b32_e32 v12, s12
-; GFX10-NEXT: v_mov_b32_e32 v13, s13
-; GFX10-NEXT: v_mov_b32_e32 v14, s14
-; GFX10-NEXT: v_mov_b32_e32 v15, s15
-; GFX10-NEXT: v_mov_b32_e32 v16, s16
-; GFX10-NEXT: v_mov_b32_e32 v17, s17
+; GFX10-NEXT: v_mov_b32_e32 v17, s15
+; GFX10-NEXT: v_mov_b32_e32 v16, s14
+; GFX10-NEXT: v_mov_b32_e32 v15, s13
+; GFX10-NEXT: v_mov_b32_e32 v14, s12
+; GFX10-NEXT: v_mov_b32_e32 v13, s11
+; GFX10-NEXT: v_mov_b32_e32 v12, s10
+; GFX10-NEXT: v_mov_b32_e32 v11, s9
+; GFX10-NEXT: v_mov_b32_e32 v10, s8
+; GFX10-NEXT: v_mov_b32_e32 v9, s7
+; GFX10-NEXT: v_mov_b32_e32 v8, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s4
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_movreld_b32_e32 v2, v0
; GFX10-NEXT: v_movreld_b32_e32 v3, v1
; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
@@ -1479,15 +1511,31 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_s(<8 x double> inreg %vec, do
;
; GFX11-LABEL: dyn_insertelement_v8f64_s_v_s:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: s_mov_b32 s3, s5
+; GFX11-NEXT: s_mov_b32 s4, s6
+; GFX11-NEXT: s_mov_b32 s5, s7
+; GFX11-NEXT: s_mov_b32 s6, s8
+; GFX11-NEXT: s_mov_b32 s7, s9
+; GFX11-NEXT: s_mov_b32 s8, s10
+; GFX11-NEXT: s_mov_b32 s9, s11
+; GFX11-NEXT: s_mov_b32 s10, s12
+; GFX11-NEXT: s_mov_b32 s11, s13
+; GFX11-NEXT: s_mov_b32 s12, s14
+; GFX11-NEXT: s_mov_b32 s13, s15
+; GFX11-NEXT: s_mov_b32 s14, s16
+; GFX11-NEXT: s_mov_b32 s15, s17
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_lshl_b32 m0, s18, 1
-; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
-; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
-; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
-; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
-; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
-; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX11-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX11-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
+; GFX11-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX11-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX11-NEXT: v_movreld_b32_e32 v2, v0
; GFX11-NEXT: v_movreld_b32_e32 v3, v1
; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off dlc
@@ -1502,22 +1550,38 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_s(<8 x double> inreg %vec, do
;
; GFX8-LABEL: dyn_insertelement_v8f64_s_v_s:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v7, s7
-; GFX8-NEXT: v_mov_b32_e32 v8, s8
-; GFX8-NEXT: v_mov_b32_e32 v9, s9
-; GFX8-NEXT: v_mov_b32_e32 v10, s10
-; GFX8-NEXT: v_mov_b32_e32 v11, s11
-; GFX8-NEXT: v_mov_b32_e32 v12, s12
-; GFX8-NEXT: v_mov_b32_e32 v13, s13
-; GFX8-NEXT: v_mov_b32_e32 v14, s14
-; GFX8-NEXT: v_mov_b32_e32 v15, s15
-; GFX8-NEXT: v_mov_b32_e32 v16, s16
-; GFX8-NEXT: v_mov_b32_e32 v17, s17
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: s_mov_b32 s4, s6
+; GFX8-NEXT: s_mov_b32 s5, s7
+; GFX8-NEXT: s_mov_b32 s6, s8
+; GFX8-NEXT: s_mov_b32 s7, s9
+; GFX8-NEXT: s_mov_b32 s8, s10
+; GFX8-NEXT: s_mov_b32 s9, s11
+; GFX8-NEXT: s_mov_b32 s10, s12
+; GFX8-NEXT: s_mov_b32 s11, s13
+; GFX8-NEXT: s_mov_b32 s12, s14
+; GFX8-NEXT: s_mov_b32 s13, s15
+; GFX8-NEXT: s_mov_b32 s14, s16
+; GFX8-NEXT: s_mov_b32 s15, s17
+; GFX8-NEXT: v_mov_b32_e32 v17, s15
+; GFX8-NEXT: v_mov_b32_e32 v16, s14
+; GFX8-NEXT: v_mov_b32_e32 v15, s13
+; GFX8-NEXT: v_mov_b32_e32 v14, s12
+; GFX8-NEXT: v_mov_b32_e32 v13, s11
+; GFX8-NEXT: v_mov_b32_e32 v12, s10
+; GFX8-NEXT: v_mov_b32_e32 v11, s9
+; GFX8-NEXT: v_mov_b32_e32 v10, s8
+; GFX8-NEXT: v_mov_b32_e32 v9, s7
+; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_lshl_b32 m0, s18, 1
; GFX8-NEXT: v_movreld_b32_e32 v2, v0
; GFX8-NEXT: v_movreld_b32_e32 v3, v1
@@ -1551,9 +1615,9 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_s(<8 x double> %vec, double i
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
; GPRIDX-NEXT: v_mov_b32_e32 v0, v16
; GPRIDX-NEXT: s_set_gpr_idx_off
-; GPRIDX-NEXT: v_mov_b32_e32 v16, s3
+; GPRIDX-NEXT: v_mov_b32_e32 v17, s3
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
-; GPRIDX-NEXT: v_mov_b32_e32 v1, v16
+; GPRIDX-NEXT: v_mov_b32_e32 v1, v17
; GPRIDX-NEXT: s_set_gpr_idx_off
; GPRIDX-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GPRIDX-NEXT: s_waitcnt vmcnt(0)
@@ -1597,11 +1661,11 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_s(<8 x double> %vec, double i
;
; GFX8-LABEL: dyn_insertelement_v8f64_v_s_s:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_lshl_b32 m0, s4, 1
; GFX8-NEXT: v_mov_b32_e32 v16, s2
+; GFX8-NEXT: s_lshl_b32 m0, s4, 1
+; GFX8-NEXT: v_mov_b32_e32 v17, s3
; GFX8-NEXT: v_movreld_b32_e32 v0, v16
-; GFX8-NEXT: v_mov_b32_e32 v16, s3
-; GFX8-NEXT: v_movreld_b32_e32 v1, v16
+; GFX8-NEXT: v_movreld_b32_e32 v1, v17
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
@@ -5984,40 +6048,72 @@ entry:
define amdgpu_ps <16 x i64> @dyn_insertelement_v16i64_s_v_s(<16 x i64> inreg %vec, i64 %val, i32 inreg %idx) {
; GPRIDX-LABEL: dyn_insertelement_v16i64_s_v_s:
; GPRIDX: ; %bb.0: ; %entry
-; GPRIDX-NEXT: v_mov_b32_e32 v2, s2
-; GPRIDX-NEXT: v_mov_b32_e32 v3, s3
-; GPRIDX-NEXT: v_mov_b32_e32 v4, s4
-; GPRIDX-NEXT: v_mov_b32_e32 v5, s5
-; GPRIDX-NEXT: v_mov_b32_e32 v6, s6
-; GPRIDX-NEXT: v_mov_b32_e32 v7, s7
-; GPRIDX-NEXT: v_mov_b32_e32 v8, s8
-; GPRIDX-NEXT: v_mov_b32_e32 v9, s9
-; GPRIDX-NEXT: v_mov_b32_e32 v10, s10
-; GPRIDX-NEXT: v_mov_b32_e32 v11, s11
-; GPRIDX-NEXT: v_mov_b32_e32 v12, s12
-; GPRIDX-NEXT: v_mov_b32_e32 v13, s13
-; GPRIDX-NEXT: v_mov_b32_e32 v14, s14
-; GPRIDX-NEXT: v_mov_b32_e32 v15, s15
-; GPRIDX-NEXT: v_mov_b32_e32 v16, s16
-; GPRIDX-NEXT: v_mov_b32_e32 v17, s17
-; GPRIDX-NEXT: v_mov_b32_e32 v18, s18
-; GPRIDX-NEXT: v_mov_b32_e32 v19, s19
-; GPRIDX-NEXT: v_mov_b32_e32 v20, s20
-; GPRIDX-NEXT: v_mov_b32_e32 v21, s21
-; GPRIDX-NEXT: v_mov_b32_e32 v22, s22
-; GPRIDX-NEXT: v_mov_b32_e32 v23, s23
-; GPRIDX-NEXT: v_mov_b32_e32 v24, s24
-; GPRIDX-NEXT: v_mov_b32_e32 v25, s25
-; GPRIDX-NEXT: v_mov_b32_e32 v26, s26
-; GPRIDX-NEXT: v_mov_b32_e32 v27, s27
-; GPRIDX-NEXT: v_mov_b32_e32 v28, s28
-; GPRIDX-NEXT: v_mov_b32_e32 v29, s29
-; GPRIDX-NEXT: v_mov_b32_e32 v30, s30
-; GPRIDX-NEXT: v_mov_b32_e32 v31, s31
-; GPRIDX-NEXT: v_mov_b32_e32 v32, s32
-; GPRIDX-NEXT: v_mov_b32_e32 v33, s33
-; GPRIDX-NEXT: s_lshl_b32 s1, s34, 1
-; GPRIDX-NEXT: s_set_gpr_idx_on s1, gpr_idx(DST)
+; GPRIDX-NEXT: s_mov_b32 s0, s2
+; GPRIDX-NEXT: s_mov_b32 s1, s3
+; GPRIDX-NEXT: s_mov_b32 s2, s4
+; GPRIDX-NEXT: s_mov_b32 s3, s5
+; GPRIDX-NEXT: s_mov_b32 s4, s6
+; GPRIDX-NEXT: s_mov_b32 s5, s7
+; GPRIDX-NEXT: s_mov_b32 s6, s8
+; GPRIDX-NEXT: s_mov_b32 s7, s9
+; GPRIDX-NEXT: s_mov_b32 s8, s10
+; GPRIDX-NEXT: s_mov_b32 s9, s11
+; GPRIDX-NEXT: s_mov_b32 s10, s12
+; GPRIDX-NEXT: s_mov_b32 s11, s13
+; GPRIDX-NEXT: s_mov_b32 s12, s14
+; GPRIDX-NEXT: s_mov_b32 s13, s15
+; GPRIDX-NEXT: s_mov_b32 s14, s16
+; GPRIDX-NEXT: s_mov_b32 s15, s17
+; GPRIDX-NEXT: s_mov_b32 s16, s18
+; GPRIDX-NEXT: s_mov_b32 s17, s19
+; GPRIDX-NEXT: s_mov_b32 s18, s20
+; GPRIDX-NEXT: s_mov_b32 s19, s21
+; GPRIDX-NEXT: s_mov_b32 s20, s22
+; GPRIDX-NEXT: s_mov_b32 s21, s23
+; GPRIDX-NEXT: s_mov_b32 s22, s24
+; GPRIDX-NEXT: s_mov_b32 s23, s25
+; GPRIDX-NEXT: s_mov_b32 s24, s26
+; GPRIDX-NEXT: s_mov_b32 s25, s27
+; GPRIDX-NEXT: s_mov_b32 s26, s28
+; GPRIDX-NEXT: s_mov_b32 s27, s29
+; GPRIDX-NEXT: s_mov_b32 s28, s30
+; GPRIDX-NEXT: s_mov_b32 s29, s31
+; GPRIDX-NEXT: s_mov_b32 s31, s33
+; GPRIDX-NEXT: s_mov_b32 s30, s32
+; GPRIDX-NEXT: s_lshl_b32 s33, s34, 1
+; GPRIDX-NEXT: v_mov_b32_e32 v33, s31
+; GPRIDX-NEXT: v_mov_b32_e32 v32, s30
+; GPRIDX-NEXT: v_mov_b32_e32 v31, s29
+; GPRIDX-NEXT: v_mov_b32_e32 v30, s28
+; GPRIDX-NEXT: v_mov_b32_e32 v29, s27
+; GPRIDX-NEXT: v_mov_b32_e32 v28, s26
+; GPRIDX-NEXT: v_mov_b32_e32 v27, s25
+; GPRIDX-NEXT: v_mov_b32_e32 v26, s24
+; GPRIDX-NEXT: v_mov_b32_e32 v25, s23
+; GPRIDX-NEXT: v_mov_b32_e32 v24, s22
+; GPRIDX-NEXT: v_mov_b32_e32 v23, s21
+; GPRIDX-NEXT: v_mov_b32_e32 v22, s20
+; GPRIDX-NEXT: v_mov_b32_e32 v21, s19
+; GPRIDX-NEXT: v_mov_b32_e32 v20, s18
+; GPRIDX-NEXT: v_mov_b32_e32 v19, s17
+; GPRIDX-NEXT: v_mov_b32_e32 v18, s16
+; GPRIDX-NEXT: v_mov_b32_e32 v17, s15
+; GPRIDX-NEXT: v_mov_b32_e32 v16, s14
+; GPRIDX-NEXT: v_mov_b32_e32 v15, s13
+; GPRIDX-NEXT: v_mov_b32_e32 v14, s12
+; GPRIDX-NEXT: v_mov_b32_e32 v13, s11
+; GPRIDX-NEXT: v_mov_b32_e32 v12, s10
+; GPRIDX-NEXT: v_mov_b32_e32 v11, s9
+; GPRIDX-NEXT: v_mov_b32_e32 v10, s8
+; GPRIDX-NEXT: v_mov_b32_e32 v9, s7
+; GPRIDX-NEXT: v_mov_b32_e32 v8, s6
+; GPRIDX-NEXT: v_mov_b32_e32 v7, s5
+; GPRIDX-NEXT: v_mov_b32_e32 v6, s4
+; GPRIDX-NEXT: v_mov_b32_e32 v5, s3
+; GPRIDX-NEXT: v_mov_b32_e32 v4, s2
+; GPRIDX-NEXT: v_mov_b32_e32 v3, s1
+; GPRIDX-NEXT: v_mov_b32_e32 v2, s0
+; GPRIDX-NEXT: s_set_gpr_idx_on s33, gpr_idx(DST)
; GPRIDX-NEXT: v_mov_b32_e32 v2, v0
; GPRIDX-NEXT: v_mov_b32_e32 v3, v1
; GPRIDX-NEXT: s_set_gpr_idx_off
@@ -6057,39 +6153,71 @@ define amdgpu_ps <16 x i64> @dyn_insertelement_v16i64_s_v_s(<16 x i64> inreg %ve
;
; GFX10-LABEL: dyn_insertelement_v16i64_s_v_s:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: s_mov_b32 s8, s10
+; GFX10-NEXT: s_mov_b32 s9, s11
+; GFX10-NEXT: s_mov_b32 s10, s12
+; GFX10-NEXT: s_mov_b32 s11, s13
+; GFX10-NEXT: s_mov_b32 s12, s14
+; GFX10-NEXT: s_mov_b32 s13, s15
+; GFX10-NEXT: s_mov_b32 s14, s16
+; GFX10-NEXT: s_mov_b32 s15, s17
+; GFX10-NEXT: s_mov_b32 s16, s18
+; GFX10-NEXT: s_mov_b32 s17, s19
+; GFX10-NEXT: s_mov_b32 s18, s20
+; GFX10-NEXT: s_mov_b32 s19, s21
+; GFX10-NEXT: s_mov_b32 s20, s22
+; GFX10-NEXT: s_mov_b32 s21, s23
+; GFX10-NEXT: s_mov_b32 s22, s24
+; GFX10-NEXT: s_mov_b32 s23, s25
+; GFX10-NEXT: s_mov_b32 s24, s26
+; GFX10-NEXT: s_mov_b32 s25, s27
+; GFX10-NEXT: s_mov_b32 s26, s28
+; GFX10-NEXT: s_mov_b32 s27, s29
+; GFX10-NEXT: s_mov_b32 s28, s30
+; GFX10-NEXT: s_mov_b32 s29, s31
+; GFX10-NEXT: s_mov_b32 s31, s33
+; GFX10-NEXT: s_mov_b32 s30, s32
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_lshl_b32 m0, s34, 1
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: v_mov_b32_e32 v8, s8
-; GFX10-NEXT: v_mov_b32_e32 v9, s9
-; GFX10-NEXT: v_mov_b32_e32 v10, s10
-; GFX10-NEXT: v_mov_b32_e32 v11, s11
-; GFX10-NEXT: v_mov_b32_e32 v12, s12
-; GFX10-NEXT: v_mov_b32_e32 v13, s13
-; GFX10-NEXT: v_mov_b32_e32 v14, s14
-; GFX10-NEXT: v_mov_b32_e32 v15, s15
-; GFX10-NEXT: v_mov_b32_e32 v16, s16
-; GFX10-NEXT: v_mov_b32_e32 v17, s17
-; GFX10-NEXT: v_mov_b32_e32 v18, s18
-; GFX10-NEXT: v_mov_b32_e32 v19, s19
-; GFX10-NEXT: v_mov_b32_e32 v20, s20
-; GFX10-NEXT: v_mov_b32_e32 v21, s21
-; GFX10-NEXT: v_mov_b32_e32 v22, s22
-; GFX10-NEXT: v_mov_b32_e32 v23, s23
-; GFX10-NEXT: v_mov_b32_e32 v24, s24
-; GFX10-NEXT: v_mov_b32_e32 v25, s25
-; GFX10-NEXT: v_mov_b32_e32 v26, s26
-; GFX10-NEXT: v_mov_b32_e32 v27, s27
-; GFX10-NEXT: v_mov_b32_e32 v28, s28
-; GFX10-NEXT: v_mov_b32_e32 v29, s29
-; GFX10-NEXT: v_mov_b32_e32 v30, s30
-; GFX10-NEXT: v_mov_b32_e32 v31, s31
-; GFX10-NEXT: v_mov_b32_e32 v32, s32
-; GFX10-NEXT: v_mov_b32_e32 v33, s33
+; GFX10-NEXT: v_mov_b32_e32 v33, s31
+; GFX10-NEXT: v_mov_b32_e32 v32, s30
+; GFX10-NEXT: v_mov_b32_e32 v31, s29
+; GFX10-NEXT: v_mov_b32_e32 v30, s28
+; GFX10-NEXT: v_mov_b32_e32 v29, s27
+; GFX10-NEXT: v_mov_b32_e32 v28, s26
+; GFX10-NEXT: v_mov_b32_e32 v27, s25
+; GFX10-NEXT: v_mov_b32_e32 v26, s24
+; GFX10-NEXT: v_mov_b32_e32 v25, s23
+; GFX10-NEXT: v_mov_b32_e32 v24, s22
+; GFX10-NEXT: v_mov_b32_e32 v23, s21
+; GFX10-NEXT: v_mov_b32_e32 v22, s20
+; GFX10-NEXT: v_mov_b32_e32 v21, s19
+; GFX10-NEXT: v_mov_b32_e32 v20, s18
+; GFX10-NEXT: v_mov_b32_e32 v19, s17
+; GFX10-NEXT: v_mov_b32_e32 v18, s16
+; GFX10-NEXT: v_mov_b32_e32 v17, s15
+; GFX10-NEXT: v_mov_b32_e32 v16, s14
+; GFX10-NEXT: v_mov_b32_e32 v15, s13
+; GFX10-NEXT: v_mov_b32_e32 v14, s12
+; GFX10-NEXT: v_mov_b32_e32 v13, s11
+; GFX10-NEXT: v_mov_b32_e32 v12, s10
+; GFX10-NEXT: v_mov_b32_e32 v11, s9
+; GFX10-NEXT: v_mov_b32_e32 v10, s8
+; GFX10-NEXT: v_mov_b32_e32 v9, s7
+; GFX10-NEXT: v_mov_b32_e32 v8, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s4
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_movreld_b32_e32 v2, v0
; GFX10-NEXT: v_movreld_b32_e32 v3, v1
; GFX10-NEXT: v_readfirstlane_b32 s0, v2
@@ -6128,23 +6256,55 @@ define amdgpu_ps <16 x i64> @dyn_insertelement_v16i64_s_v_s(<16 x i64> inreg %ve
;
; GFX11-LABEL: dyn_insertelement_v16i64_s_v_s:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: s_mov_b32 s3, s5
+; GFX11-NEXT: s_mov_b32 s4, s6
+; GFX11-NEXT: s_mov_b32 s5, s7
+; GFX11-NEXT: s_mov_b32 s6, s8
+; GFX11-NEXT: s_mov_b32 s7, s9
+; GFX11-NEXT: s_mov_b32 s8, s10
+; GFX11-NEXT: s_mov_b32 s9, s11
+; GFX11-NEXT: s_mov_b32 s10, s12
+; GFX11-NEXT: s_mov_b32 s11, s13
+; GFX11-NEXT: s_mov_b32 s12, s14
+; GFX11-NEXT: s_mov_b32 s13, s15
+; GFX11-NEXT: s_mov_b32 s14, s16
+; GFX11-NEXT: s_mov_b32 s15, s17
+; GFX11-NEXT: s_mov_b32 s16, s18
+; GFX11-NEXT: s_mov_b32 s17, s19
+; GFX11-NEXT: s_mov_b32 s18, s20
+; GFX11-NEXT: s_mov_b32 s19, s21
+; GFX11-NEXT: s_mov_b32 s20, s22
+; GFX11-NEXT: s_mov_b32 s21, s23
+; GFX11-NEXT: s_mov_b32 s22, s24
+; GFX11-NEXT: s_mov_b32 s23, s25
+; GFX11-NEXT: s_mov_b32 s24, s26
+; GFX11-NEXT: s_mov_b32 s25, s27
+; GFX11-NEXT: s_mov_b32 s26, s28
+; GFX11-NEXT: s_mov_b32 s27, s29
+; GFX11-NEXT: s_mov_b32 s28, s30
+; GFX11-NEXT: s_mov_b32 s29, s31
+; GFX11-NEXT: s_mov_b32 s31, s33
+; GFX11-NEXT: s_mov_b32 s30, s32
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_lshl_b32 m0, s34, 1
-; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
-; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
-; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
-; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
-; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
-; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
-; GFX11-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
-; GFX11-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
-; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
-; GFX11-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
-; GFX11-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
-; GFX11-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
-; GFX11-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
-; GFX11-NEXT: v_dual_mov_b32 v32, s32 :: v_dual_mov_b32 v33, s33
+; GFX11-NEXT: v_dual_mov_b32 v33, s31 :: v_dual_mov_b32 v32, s30
+; GFX11-NEXT: v_dual_mov_b32 v31, s29 :: v_dual_mov_b32 v30, s28
+; GFX11-NEXT: v_dual_mov_b32 v29, s27 :: v_dual_mov_b32 v28, s26
+; GFX11-NEXT: v_dual_mov_b32 v27, s25 :: v_dual_mov_b32 v26, s24
+; GFX11-NEXT: v_dual_mov_b32 v25, s23 :: v_dual_mov_b32 v24, s22
+; GFX11-NEXT: v_dual_mov_b32 v23, s21 :: v_dual_mov_b32 v22, s20
+; GFX11-NEXT: v_dual_mov_b32 v21, s19 :: v_dual_mov_b32 v20, s18
+; GFX11-NEXT: v_dual_mov_b32 v19, s17 :: v_dual_mov_b32 v18, s16
+; GFX11-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX11-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
+; GFX11-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX11-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX11-NEXT: v_movreld_b32_e32 v2, v0
; GFX11-NEXT: v_movreld_b32_e32 v3, v1
; GFX11-NEXT: v_readfirstlane_b32 s0, v2
@@ -6183,39 +6343,71 @@ define amdgpu_ps <16 x i64> @dyn_insertelement_v16i64_s_v_s(<16 x i64> inreg %ve
;
; GFX8-LABEL: dyn_insertelement_v16i64_s_v_s:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v7, s7
-; GFX8-NEXT: v_mov_b32_e32 v8, s8
-; GFX8-NEXT: v_mov_b32_e32 v9, s9
-; GFX8-NEXT: v_mov_b32_e32 v10, s10
-; GFX8-NEXT: v_mov_b32_e32 v11, s11
-; GFX8-NEXT: v_mov_b32_e32 v12, s12
-; GFX8-NEXT: v_mov_b32_e32 v13, s13
-; GFX8-NEXT: v_mov_b32_e32 v14, s14
-; GFX8-NEXT: v_mov_b32_e32 v15, s15
-; GFX8-NEXT: v_mov_b32_e32 v16, s16
-; GFX8-NEXT: v_mov_b32_e32 v17, s17
-; GFX8-NEXT: v_mov_b32_e32 v18, s18
-; GFX8-NEXT: v_mov_b32_e32 v19, s19
-; GFX8-NEXT: v_mov_b32_e32 v20, s20
-; GFX8-NEXT: v_mov_b32_e32 v21, s21
-; GFX8-NEXT: v_mov_b32_e32 v22, s22
-; GFX8-NEXT: v_mov_b32_e32 v23, s23
-; GFX8-NEXT: v_mov_b32_e32 v24, s24
-; GFX8-NEXT: v_mov_b32_e32 v25, s25
-; GFX8-NEXT: v_mov_b32_e32 v26, s26
-; GFX8-NEXT: v_mov_b32_e32 v27, s27
-; GFX8-NEXT: v_mov_b32_e32 v28, s28
-; GFX8-NEXT: v_mov_b32_e32 v29, s29
-; GFX8-NEXT: v_mov_b32_e32 v30, s30
-; GFX8-NEXT: v_mov_b32_e32 v31, s31
-; GFX8-NEXT: v_mov_b32_e32 v32, s32
-; GFX8-NEXT: v_mov_b32_e32 v33, s33
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: s_mov_b32 s4, s6
+; GFX8-NEXT: s_mov_b32 s5, s7
+; GFX8-NEXT: s_mov_b32 s6, s8
+; GFX8-NEXT: s_mov_b32 s7, s9
+; GFX8-NEXT: s_mov_b32 s8, s10
+; GFX8-NEXT: s_mov_b32 s9, s11
+; GFX8-NEXT: s_mov_b32 s10, s12
+; GFX8-NEXT: s_mov_b32 s11, s13
+; GFX8-NEXT: s_mov_b32 s12, s14
+; GFX8-NEXT: s_mov_b32 s13, s15
+; GFX8-NEXT: s_mov_b32 s14, s16
+; GFX8-NEXT: s_mov_b32 s15, s17
+; GFX8-NEXT: s_mov_b32 s16, s18
+; GFX8-NEXT: s_mov_b32 s17, s19
+; GFX8-NEXT: s_mov_b32 s18, s20
+; GFX8-NEXT: s_mov_b32 s19, s21
+; GFX8-NEXT: s_mov_b32 s20, s22
+; GFX8-NEXT: s_mov_b32 s21, s23
+; GFX8-NEXT: s_mov_b32 s22, s24
+; GFX8-NEXT: s_mov_b32 s23, s25
+; GFX8-NEXT: s_mov_b32 s24, s26
+; GFX8-NEXT: s_mov_b32 s25, s27
+; GFX8-NEXT: s_mov_b32 s26, s28
+; GFX8-NEXT: s_mov_b32 s27, s29
+; GFX8-NEXT: s_mov_b32 s28, s30
+; GFX8-NEXT: s_mov_b32 s29, s31
+; GFX8-NEXT: s_mov_b32 s31, s33
+; GFX8-NEXT: s_mov_b32 s30, s32
; GFX8-NEXT: s_lshl_b32 m0, s34, 1
+; GFX8-NEXT: v_mov_b32_e32 v33, s31
+; GFX8-NEXT: v_mov_b32_e32 v32, s30
+; GFX8-NEXT: v_mov_b32_e32 v31, s29
+; GFX8-NEXT: v_mov_b32_e32 v30, s28
+; GFX8-NEXT: v_mov_b32_e32 v29, s27
+; GFX8-NEXT: v_mov_b32_e32 v28, s26
+; GFX8-NEXT: v_mov_b32_e32 v27, s25
+; GFX8-NEXT: v_mov_b32_e32 v26, s24
+; GFX8-NEXT: v_mov_b32_e32 v25, s23
+; GFX8-NEXT: v_mov_b32_e32 v24, s22
+; GFX8-NEXT: v_mov_b32_e32 v23, s21
+; GFX8-NEXT: v_mov_b32_e32 v22, s20
+; GFX8-NEXT: v_mov_b32_e32 v21, s19
+; GFX8-NEXT: v_mov_b32_e32 v20, s18
+; GFX8-NEXT: v_mov_b32_e32 v19, s17
+; GFX8-NEXT: v_mov_b32_e32 v18, s16
+; GFX8-NEXT: v_mov_b32_e32 v17, s15
+; GFX8-NEXT: v_mov_b32_e32 v16, s14
+; GFX8-NEXT: v_mov_b32_e32 v15, s13
+; GFX8-NEXT: v_mov_b32_e32 v14, s12
+; GFX8-NEXT: v_mov_b32_e32 v13, s11
+; GFX8-NEXT: v_mov_b32_e32 v12, s10
+; GFX8-NEXT: v_mov_b32_e32 v11, s9
+; GFX8-NEXT: v_mov_b32_e32 v10, s8
+; GFX8-NEXT: v_mov_b32_e32 v9, s7
+; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_movreld_b32_e32 v2, v0
; GFX8-NEXT: v_movreld_b32_e32 v3, v1
; GFX8-NEXT: v_readfirstlane_b32 s0, v2
@@ -6259,40 +6451,72 @@ entry:
define amdgpu_ps <16 x double> @dyn_insertelement_v16f64_s_v_s(<16 x double> inreg %vec, double %val, i32 inreg %idx) {
; GPRIDX-LABEL: dyn_insertelement_v16f64_s_v_s:
; GPRIDX: ; %bb.0: ; %entry
-; GPRIDX-NEXT: v_mov_b32_e32 v2, s2
-; GPRIDX-NEXT: v_mov_b32_e32 v3, s3
-; GPRIDX-NEXT: v_mov_b32_e32 v4, s4
-; GPRIDX-NEXT: v_mov_b32_e32 v5, s5
-; GPRIDX-NEXT: v_mov_b32_e32 v6, s6
-; GPRIDX-NEXT: v_mov_b32_e32 v7, s7
-; GPRIDX-NEXT: v_mov_b32_e32 v8, s8
-; GPRIDX-NEXT: v_mov_b32_e32 v9, s9
-; GPRIDX-NEXT: v_mov_b32_e32 v10, s10
-; GPRIDX-NEXT: v_mov_b32_e32 v11, s11
-; GPRIDX-NEXT: v_mov_b32_e32 v12, s12
-; GPRIDX-NEXT: v_mov_b32_e32 v13, s13
-; GPRIDX-NEXT: v_mov_b32_e32 v14, s14
-; GPRIDX-NEXT: v_mov_b32_e32 v15, s15
-; GPRIDX-NEXT: v_mov_b32_e32 v16, s16
-; GPRIDX-NEXT: v_mov_b32_e32 v17, s17
-; GPRIDX-NEXT: v_mov_b32_e32 v18, s18
-; GPRIDX-NEXT: v_mov_b32_e32 v19, s19
-; GPRIDX-NEXT: v_mov_b32_e32 v20, s20
-; GPRIDX-NEXT: v_mov_b32_e32 v21, s21
-; GPRIDX-NEXT: v_mov_b32_e32 v22, s22
-; GPRIDX-NEXT: v_mov_b32_e32 v23, s23
-; GPRIDX-NEXT: v_mov_b32_e32 v24, s24
-; GPRIDX-NEXT: v_mov_b32_e32 v25, s25
-; GPRIDX-NEXT: v_mov_b32_e32 v26, s26
-; GPRIDX-NEXT: v_mov_b32_e32 v27, s27
-; GPRIDX-NEXT: v_mov_b32_e32 v28, s28
-; GPRIDX-NEXT: v_mov_b32_e32 v29, s29
-; GPRIDX-NEXT: v_mov_b32_e32 v30, s30
-; GPRIDX-NEXT: v_mov_b32_e32 v31, s31
-; GPRIDX-NEXT: v_mov_b32_e32 v32, s32
-; GPRIDX-NEXT: v_mov_b32_e32 v33, s33
-; GPRIDX-NEXT: s_lshl_b32 s1, s34, 1
-; GPRIDX-NEXT: s_set_gpr_idx_on s1, gpr_idx(DST)
+; GPRIDX-NEXT: s_mov_b32 s0, s2
+; GPRIDX-NEXT: s_mov_b32 s1, s3
+; GPRIDX-NEXT: s_mov_b32 s2, s4
+; GPRIDX-NEXT: s_mov_b32 s3, s5
+; GPRIDX-NEXT: s_mov_b32 s4, s6
+; GPRIDX-NEXT: s_mov_b32 s5, s7
+; GPRIDX-NEXT: s_mov_b32 s6, s8
+; GPRIDX-NEXT: s_mov_b32 s7, s9
+; GPRIDX-NEXT: s_mov_b32 s8, s10
+; GPRIDX-NEXT: s_mov_b32 s9, s11
+; GPRIDX-NEXT: s_mov_b32 s10, s12
+; GPRIDX-NEXT: s_mov_b32 s11, s13
+; GPRIDX-NEXT: s_mov_b32 s12, s14
+; GPRIDX-NEXT: s_mov_b32 s13, s15
+; GPRIDX-NEXT: s_mov_b32 s14, s16
+; GPRIDX-NEXT: s_mov_b32 s15, s17
+; GPRIDX-NEXT: s_mov_b32 s16, s18
+; GPRIDX-NEXT: s_mov_b32 s17, s19
+; GPRIDX-NEXT: s_mov_b32 s18, s20
+; GPRIDX-NEXT: s_mov_b32 s19, s21
+; GPRIDX-NEXT: s_mov_b32 s20, s22
+; GPRIDX-NEXT: s_mov_b32 s21, s23
+; GPRIDX-NEXT: s_mov_b32 s22, s24
+; GPRIDX-NEXT: s_mov_b32 s23, s25
+; GPRIDX-NEXT: s_mov_b32 s24, s26
+; GPRIDX-NEXT: s_mov_b32 s25, s27
+; GPRIDX-NEXT: s_mov_b32 s26, s28
+; GPRIDX-NEXT: s_mov_b32 s27, s29
+; GPRIDX-NEXT: s_mov_b32 s28, s30
+; GPRIDX-NEXT: s_mov_b32 s29, s31
+; GPRIDX-NEXT: s_mov_b32 s31, s33
+; GPRIDX-NEXT: s_mov_b32 s30, s32
+; GPRIDX-NEXT: s_lshl_b32 s33, s34, 1
+; GPRIDX-NEXT: v_mov_b32_e32 v33, s31
+; GPRIDX-NEXT: v_mov_b32_e32 v32, s30
+; GPRIDX-NEXT: v_mov_b32_e32 v31, s29
+; GPRIDX-NEXT: v_mov_b32_e32 v30, s28
+; GPRIDX-NEXT: v_mov_b32_e32 v29, s27
+; GPRIDX-NEXT: v_mov_b32_e32 v28, s26
+; GPRIDX-NEXT: v_mov_b32_e32 v27, s25
+; GPRIDX-NEXT: v_mov_b32_e32 v26, s24
+; GPRIDX-NEXT: v_mov_b32_e32 v25, s23
+; GPRIDX-NEXT: v_mov_b32_e32 v24, s22
+; GPRIDX-NEXT: v_mov_b32_e32 v23, s21
+; GPRIDX-NEXT: v_mov_b32_e32 v22, s20
+; GPRIDX-NEXT: v_mov_b32_e32 v21, s19
+; GPRIDX-NEXT: v_mov_b32_e32 v20, s18
+; GPRIDX-NEXT: v_mov_b32_e32 v19, s17
+; GPRIDX-NEXT: v_mov_b32_e32 v18, s16
+; GPRIDX-NEXT: v_mov_b32_e32 v17, s15
+; GPRIDX-NEXT: v_mov_b32_e32 v16, s14
+; GPRIDX-NEXT: v_mov_b32_e32 v15, s13
+; GPRIDX-NEXT: v_mov_b32_e32 v14, s12
+; GPRIDX-NEXT: v_mov_b32_e32 v13, s11
+; GPRIDX-NEXT: v_mov_b32_e32 v12, s10
+; GPRIDX-NEXT: v_mov_b32_e32 v11, s9
+; GPRIDX-NEXT: v_mov_b32_e32 v10, s8
+; GPRIDX-NEXT: v_mov_b32_e32 v9, s7
+; GPRIDX-NEXT: v_mov_b32_e32 v8, s6
+; GPRIDX-NEXT: v_mov_b32_e32 v7, s5
+; GPRIDX-NEXT: v_mov_b32_e32 v6, s4
+; GPRIDX-NEXT: v_mov_b32_e32 v5, s3
+; GPRIDX-NEXT: v_mov_b32_e32 v4, s2
+; GPRIDX-NEXT: v_mov_b32_e32 v3, s1
+; GPRIDX-NEXT: v_mov_b32_e32 v2, s0
+; GPRIDX-NEXT: s_set_gpr_idx_on s33, gpr_idx(DST)
; GPRIDX-NEXT: v_mov_b32_e32 v2, v0
; GPRIDX-NEXT: v_mov_b32_e32 v3, v1
; GPRIDX-NEXT: s_set_gpr_idx_off
@@ -6332,39 +6556,71 @@ define amdgpu_ps <16 x double> @dyn_insertelement_v16f64_s_v_s(<16 x double> inr
;
; GFX10-LABEL: dyn_insertelement_v16f64_s_v_s:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: s_mov_b32 s8, s10
+; GFX10-NEXT: s_mov_b32 s9, s11
+; GFX10-NEXT: s_mov_b32 s10, s12
+; GFX10-NEXT: s_mov_b32 s11, s13
+; GFX10-NEXT: s_mov_b32 s12, s14
+; GFX10-NEXT: s_mov_b32 s13, s15
+; GFX10-NEXT: s_mov_b32 s14, s16
+; GFX10-NEXT: s_mov_b32 s15, s17
+; GFX10-NEXT: s_mov_b32 s16, s18
+; GFX10-NEXT: s_mov_b32 s17, s19
+; GFX10-NEXT: s_mov_b32 s18, s20
+; GFX10-NEXT: s_mov_b32 s19, s21
+; GFX10-NEXT: s_mov_b32 s20, s22
+; GFX10-NEXT: s_mov_b32 s21, s23
+; GFX10-NEXT: s_mov_b32 s22, s24
+; GFX10-NEXT: s_mov_b32 s23, s25
+; GFX10-NEXT: s_mov_b32 s24, s26
+; GFX10-NEXT: s_mov_b32 s25, s27
+; GFX10-NEXT: s_mov_b32 s26, s28
+; GFX10-NEXT: s_mov_b32 s27, s29
+; GFX10-NEXT: s_mov_b32 s28, s30
+; GFX10-NEXT: s_mov_b32 s29, s31
+; GFX10-NEXT: s_mov_b32 s31, s33
+; GFX10-NEXT: s_mov_b32 s30, s32
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_lshl_b32 m0, s34, 1
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: v_mov_b32_e32 v8, s8
-; GFX10-NEXT: v_mov_b32_e32 v9, s9
-; GFX10-NEXT: v_mov_b32_e32 v10, s10
-; GFX10-NEXT: v_mov_b32_e32 v11, s11
-; GFX10-NEXT: v_mov_b32_e32 v12, s12
-; GFX10-NEXT: v_mov_b32_e32 v13, s13
-; GFX10-NEXT: v_mov_b32_e32 v14, s14
-; GFX10-NEXT: v_mov_b32_e32 v15, s15
-; GFX10-NEXT: v_mov_b32_e32 v16, s16
-; GFX10-NEXT: v_mov_b32_e32 v17, s17
-; GFX10-NEXT: v_mov_b32_e32 v18, s18
-; GFX10-NEXT: v_mov_b32_e32 v19, s19
-; GFX10-NEXT: v_mov_b32_e32 v20, s20
-; GFX10-NEXT: v_mov_b32_e32 v21, s21
-; GFX10-NEXT: v_mov_b32_e32 v22, s22
-; GFX10-NEXT: v_mov_b32_e32 v23, s23
-; GFX10-NEXT: v_mov_b32_e32 v24, s24
-; GFX10-NEXT: v_mov_b32_e32 v25, s25
-; GFX10-NEXT: v_mov_b32_e32 v26, s26
-; GFX10-NEXT: v_mov_b32_e32 v27, s27
-; GFX10-NEXT: v_mov_b32_e32 v28, s28
-; GFX10-NEXT: v_mov_b32_e32 v29, s29
-; GFX10-NEXT: v_mov_b32_e32 v30, s30
-; GFX10-NEXT: v_mov_b32_e32 v31, s31
-; GFX10-NEXT: v_mov_b32_e32 v32, s32
-; GFX10-NEXT: v_mov_b32_e32 v33, s33
+; GFX10-NEXT: v_mov_b32_e32 v33, s31
+; GFX10-NEXT: v_mov_b32_e32 v32, s30
+; GFX10-NEXT: v_mov_b32_e32 v31, s29
+; GFX10-NEXT: v_mov_b32_e32 v30, s28
+; GFX10-NEXT: v_mov_b32_e32 v29, s27
+; GFX10-NEXT: v_mov_b32_e32 v28, s26
+; GFX10-NEXT: v_mov_b32_e32 v27, s25
+; GFX10-NEXT: v_mov_b32_e32 v26, s24
+; GFX10-NEXT: v_mov_b32_e32 v25, s23
+; GFX10-NEXT: v_mov_b32_e32 v24, s22
+; GFX10-NEXT: v_mov_b32_e32 v23, s21
+; GFX10-NEXT: v_mov_b32_e32 v22, s20
+; GFX10-NEXT: v_mov_b32_e32 v21, s19
+; GFX10-NEXT: v_mov_b32_e32 v20, s18
+; GFX10-NEXT: v_mov_b32_e32 v19, s17
+; GFX10-NEXT: v_mov_b32_e32 v18, s16
+; GFX10-NEXT: v_mov_b32_e32 v17, s15
+; GFX10-NEXT: v_mov_b32_e32 v16, s14
+; GFX10-NEXT: v_mov_b32_e32 v15, s13
+; GFX10-NEXT: v_mov_b32_e32 v14, s12
+; GFX10-NEXT: v_mov_b32_e32 v13, s11
+; GFX10-NEXT: v_mov_b32_e32 v12, s10
+; GFX10-NEXT: v_mov_b32_e32 v11, s9
+; GFX10-NEXT: v_mov_b32_e32 v10, s8
+; GFX10-NEXT: v_mov_b32_e32 v9, s7
+; GFX10-NEXT: v_mov_b32_e32 v8, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s4
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_movreld_b32_e32 v2, v0
; GFX10-NEXT: v_movreld_b32_e32 v3, v1
; GFX10-NEXT: v_readfirstlane_b32 s0, v2
@@ -6403,23 +6659,55 @@ define amdgpu_ps <16 x double> @dyn_insertelement_v16f64_s_v_s(<16 x double> inr
;
; GFX11-LABEL: dyn_insertelement_v16f64_s_v_s:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: s_mov_b32 s3, s5
+; GFX11-NEXT: s_mov_b32 s4, s6
+; GFX11-NEXT: s_mov_b32 s5, s7
+; GFX11-NEXT: s_mov_b32 s6, s8
+; GFX11-NEXT: s_mov_b32 s7, s9
+; GFX11-NEXT: s_mov_b32 s8, s10
+; GFX11-NEXT: s_mov_b32 s9, s11
+; GFX11-NEXT: s_mov_b32 s10, s12
+; GFX11-NEXT: s_mov_b32 s11, s13
+; GFX11-NEXT: s_mov_b32 s12, s14
+; GFX11-NEXT: s_mov_b32 s13, s15
+; GFX11-NEXT: s_mov_b32 s14, s16
+; GFX11-NEXT: s_mov_b32 s15, s17
+; GFX11-NEXT: s_mov_b32 s16, s18
+; GFX11-NEXT: s_mov_b32 s17, s19
+; GFX11-NEXT: s_mov_b32 s18, s20
+; GFX11-NEXT: s_mov_b32 s19, s21
+; GFX11-NEXT: s_mov_b32 s20, s22
+; GFX11-NEXT: s_mov_b32 s21, s23
+; GFX11-NEXT: s_mov_b32 s22, s24
+; GFX11-NEXT: s_mov_b32 s23, s25
+; GFX11-NEXT: s_mov_b32 s24, s26
+; GFX11-NEXT: s_mov_b32 s25, s27
+; GFX11-NEXT: s_mov_b32 s26, s28
+; GFX11-NEXT: s_mov_b32 s27, s29
+; GFX11-NEXT: s_mov_b32 s28, s30
+; GFX11-NEXT: s_mov_b32 s29, s31
+; GFX11-NEXT: s_mov_b32 s31, s33
+; GFX11-NEXT: s_mov_b32 s30, s32
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_lshl_b32 m0, s34, 1
-; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
-; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
-; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
-; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
-; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
-; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
-; GFX11-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
-; GFX11-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
-; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
-; GFX11-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
-; GFX11-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
-; GFX11-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
-; GFX11-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
-; GFX11-NEXT: v_dual_mov_b32 v32, s32 :: v_dual_mov_b32 v33, s33
+; GFX11-NEXT: v_dual_mov_b32 v33, s31 :: v_dual_mov_b32 v32, s30
+; GFX11-NEXT: v_dual_mov_b32 v31, s29 :: v_dual_mov_b32 v30, s28
+; GFX11-NEXT: v_dual_mov_b32 v29, s27 :: v_dual_mov_b32 v28, s26
+; GFX11-NEXT: v_dual_mov_b32 v27, s25 :: v_dual_mov_b32 v26, s24
+; GFX11-NEXT: v_dual_mov_b32 v25, s23 :: v_dual_mov_b32 v24, s22
+; GFX11-NEXT: v_dual_mov_b32 v23, s21 :: v_dual_mov_b32 v22, s20
+; GFX11-NEXT: v_dual_mov_b32 v21, s19 :: v_dual_mov_b32 v20, s18
+; GFX11-NEXT: v_dual_mov_b32 v19, s17 :: v_dual_mov_b32 v18, s16
+; GFX11-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX11-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
+; GFX11-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX11-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX11-NEXT: v_movreld_b32_e32 v2, v0
; GFX11-NEXT: v_movreld_b32_e32 v3, v1
; GFX11-NEXT: v_readfirstlane_b32 s0, v2
@@ -6458,39 +6746,71 @@ define amdgpu_ps <16 x double> @dyn_insertelement_v16f64_s_v_s(<16 x double> inr
;
; GFX8-LABEL: dyn_insertelement_v16f64_s_v_s:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v7, s7
-; GFX8-NEXT: v_mov_b32_e32 v8, s8
-; GFX8-NEXT: v_mov_b32_e32 v9, s9
-; GFX8-NEXT: v_mov_b32_e32 v10, s10
-; GFX8-NEXT: v_mov_b32_e32 v11, s11
-; GFX8-NEXT: v_mov_b32_e32 v12, s12
-; GFX8-NEXT: v_mov_b32_e32 v13, s13
-; GFX8-NEXT: v_mov_b32_e32 v14, s14
-; GFX8-NEXT: v_mov_b32_e32 v15, s15
-; GFX8-NEXT: v_mov_b32_e32 v16, s16
-; GFX8-NEXT: v_mov_b32_e32 v17, s17
-; GFX8-NEXT: v_mov_b32_e32 v18, s18
-; GFX8-NEXT: v_mov_b32_e32 v19, s19
-; GFX8-NEXT: v_mov_b32_e32 v20, s20
-; GFX8-NEXT: v_mov_b32_e32 v21, s21
-; GFX8-NEXT: v_mov_b32_e32 v22, s22
-; GFX8-NEXT: v_mov_b32_e32 v23, s23
-; GFX8-NEXT: v_mov_b32_e32 v24, s24
-; GFX8-NEXT: v_mov_b32_e32 v25, s25
-; GFX8-NEXT: v_mov_b32_e32 v26, s26
-; GFX8-NEXT: v_mov_b32_e32 v27, s27
-; GFX8-NEXT: v_mov_b32_e32 v28, s28
-; GFX8-NEXT: v_mov_b32_e32 v29, s29
-; GFX8-NEXT: v_mov_b32_e32 v30, s30
-; GFX8-NEXT: v_mov_b32_e32 v31, s31
-; GFX8-NEXT: v_mov_b32_e32 v32, s32
-; GFX8-NEXT: v_mov_b32_e32 v33, s33
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: s_mov_b32 s4, s6
+; GFX8-NEXT: s_mov_b32 s5, s7
+; GFX8-NEXT: s_mov_b32 s6, s8
+; GFX8-NEXT: s_mov_b32 s7, s9
+; GFX8-NEXT: s_mov_b32 s8, s10
+; GFX8-NEXT: s_mov_b32 s9, s11
+; GFX8-NEXT: s_mov_b32 s10, s12
+; GFX8-NEXT: s_mov_b32 s11, s13
+; GFX8-NEXT: s_mov_b32 s12, s14
+; GFX8-NEXT: s_mov_b32 s13, s15
+; GFX8-NEXT: s_mov_b32 s14, s16
+; GFX8-NEXT: s_mov_b32 s15, s17
+; GFX8-NEXT: s_mov_b32 s16, s18
+; GFX8-NEXT: s_mov_b32 s17, s19
+; GFX8-NEXT: s_mov_b32 s18, s20
+; GFX8-NEXT: s_mov_b32 s19, s21
+; GFX8-NEXT: s_mov_b32 s20, s22
+; GFX8-NEXT: s_mov_b32 s21, s23
+; GFX8-NEXT: s_mov_b32 s22, s24
+; GFX8-NEXT: s_mov_b32 s23, s25
+; GFX8-NEXT: s_mov_b32 s24, s26
+; GFX8-NEXT: s_mov_b32 s25, s27
+; GFX8-NEXT: s_mov_b32 s26, s28
+; GFX8-NEXT: s_mov_b32 s27, s29
+; GFX8-NEXT: s_mov_b32 s28, s30
+; GFX8-NEXT: s_mov_b32 s29, s31
+; GFX8-NEXT: s_mov_b32 s31, s33
+; GFX8-NEXT: s_mov_b32 s30, s32
; GFX8-NEXT: s_lshl_b32 m0, s34, 1
+; GFX8-NEXT: v_mov_b32_e32 v33, s31
+; GFX8-NEXT: v_mov_b32_e32 v32, s30
+; GFX8-NEXT: v_mov_b32_e32 v31, s29
+; GFX8-NEXT: v_mov_b32_e32 v30, s28
+; GFX8-NEXT: v_mov_b32_e32 v29, s27
+; GFX8-NEXT: v_mov_b32_e32 v28, s26
+; GFX8-NEXT: v_mov_b32_e32 v27, s25
+; GFX8-NEXT: v_mov_b32_e32 v26, s24
+; GFX8-NEXT: v_mov_b32_e32 v25, s23
+; GFX8-NEXT: v_mov_b32_e32 v24, s22
+; GFX8-NEXT: v_mov_b32_e32 v23, s21
+; GFX8-NEXT: v_mov_b32_e32 v22, s20
+; GFX8-NEXT: v_mov_b32_e32 v21, s19
+; GFX8-NEXT: v_mov_b32_e32 v20, s18
+; GFX8-NEXT: v_mov_b32_e32 v19, s17
+; GFX8-NEXT: v_mov_b32_e32 v18, s16
+; GFX8-NEXT: v_mov_b32_e32 v17, s15
+; GFX8-NEXT: v_mov_b32_e32 v16, s14
+; GFX8-NEXT: v_mov_b32_e32 v15, s13
+; GFX8-NEXT: v_mov_b32_e32 v14, s12
+; GFX8-NEXT: v_mov_b32_e32 v13, s11
+; GFX8-NEXT: v_mov_b32_e32 v12, s10
+; GFX8-NEXT: v_mov_b32_e32 v11, s9
+; GFX8-NEXT: v_mov_b32_e32 v10, s8
+; GFX8-NEXT: v_mov_b32_e32 v9, s7
+; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_movreld_b32_e32 v2, v0
; GFX8-NEXT: v_movreld_b32_e32 v3, v1
; GFX8-NEXT: v_readfirstlane_b32 s0, v2
@@ -6950,20 +7270,36 @@ entry:
define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_s(<7 x double> inreg %vec, double %val, i32 inreg %idx) {
; GPRIDX-LABEL: dyn_insertelement_v7f64_s_v_s:
; GPRIDX: ; %bb.0: ; %entry
-; GPRIDX-NEXT: v_mov_b32_e32 v2, s2
-; GPRIDX-NEXT: v_mov_b32_e32 v3, s3
-; GPRIDX-NEXT: v_mov_b32_e32 v4, s4
-; GPRIDX-NEXT: v_mov_b32_e32 v5, s5
-; GPRIDX-NEXT: v_mov_b32_e32 v6, s6
-; GPRIDX-NEXT: v_mov_b32_e32 v7, s7
-; GPRIDX-NEXT: v_mov_b32_e32 v8, s8
-; GPRIDX-NEXT: v_mov_b32_e32 v9, s9
-; GPRIDX-NEXT: v_mov_b32_e32 v10, s10
-; GPRIDX-NEXT: v_mov_b32_e32 v11, s11
-; GPRIDX-NEXT: v_mov_b32_e32 v12, s12
-; GPRIDX-NEXT: v_mov_b32_e32 v13, s13
-; GPRIDX-NEXT: v_mov_b32_e32 v14, s14
-; GPRIDX-NEXT: v_mov_b32_e32 v15, s15
+; GPRIDX-NEXT: s_mov_b32 s0, s2
+; GPRIDX-NEXT: s_mov_b32 s1, s3
+; GPRIDX-NEXT: s_mov_b32 s2, s4
+; GPRIDX-NEXT: s_mov_b32 s3, s5
+; GPRIDX-NEXT: s_mov_b32 s4, s6
+; GPRIDX-NEXT: s_mov_b32 s5, s7
+; GPRIDX-NEXT: s_mov_b32 s6, s8
+; GPRIDX-NEXT: s_mov_b32 s7, s9
+; GPRIDX-NEXT: s_mov_b32 s8, s10
+; GPRIDX-NEXT: s_mov_b32 s9, s11
+; GPRIDX-NEXT: s_mov_b32 s10, s12
+; GPRIDX-NEXT: s_mov_b32 s11, s13
+; GPRIDX-NEXT: s_mov_b32 s12, s14
+; GPRIDX-NEXT: s_mov_b32 s13, s15
+; GPRIDX-NEXT: v_mov_b32_e32 v17, s15
+; GPRIDX-NEXT: v_mov_b32_e32 v16, s14
+; GPRIDX-NEXT: v_mov_b32_e32 v15, s13
+; GPRIDX-NEXT: v_mov_b32_e32 v14, s12
+; GPRIDX-NEXT: v_mov_b32_e32 v13, s11
+; GPRIDX-NEXT: v_mov_b32_e32 v12, s10
+; GPRIDX-NEXT: v_mov_b32_e32 v11, s9
+; GPRIDX-NEXT: v_mov_b32_e32 v10, s8
+; GPRIDX-NEXT: v_mov_b32_e32 v9, s7
+; GPRIDX-NEXT: v_mov_b32_e32 v8, s6
+; GPRIDX-NEXT: v_mov_b32_e32 v7, s5
+; GPRIDX-NEXT: v_mov_b32_e32 v6, s4
+; GPRIDX-NEXT: v_mov_b32_e32 v5, s3
+; GPRIDX-NEXT: v_mov_b32_e32 v4, s2
+; GPRIDX-NEXT: v_mov_b32_e32 v3, s1
+; GPRIDX-NEXT: v_mov_b32_e32 v2, s0
; GPRIDX-NEXT: s_lshl_b32 s0, s16, 1
; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
; GPRIDX-NEXT: v_mov_b32_e32 v2, v0
@@ -6987,21 +7323,37 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_s(<7 x double> inreg
;
; GFX10-LABEL: dyn_insertelement_v7f64_s_v_s:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: s_mov_b32 s8, s10
+; GFX10-NEXT: s_mov_b32 s9, s11
+; GFX10-NEXT: s_mov_b32 s10, s12
+; GFX10-NEXT: s_mov_b32 s11, s13
+; GFX10-NEXT: s_mov_b32 s12, s14
+; GFX10-NEXT: s_mov_b32 s13, s15
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_lshl_b32 m0, s16, 1
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: v_mov_b32_e32 v8, s8
-; GFX10-NEXT: v_mov_b32_e32 v9, s9
-; GFX10-NEXT: v_mov_b32_e32 v10, s10
-; GFX10-NEXT: v_mov_b32_e32 v11, s11
-; GFX10-NEXT: v_mov_b32_e32 v12, s12
-; GFX10-NEXT: v_mov_b32_e32 v13, s13
-; GFX10-NEXT: v_mov_b32_e32 v14, s14
-; GFX10-NEXT: v_mov_b32_e32 v15, s15
+; GFX10-NEXT: v_mov_b32_e32 v17, s15
+; GFX10-NEXT: v_mov_b32_e32 v16, s14
+; GFX10-NEXT: v_mov_b32_e32 v15, s13
+; GFX10-NEXT: v_mov_b32_e32 v14, s12
+; GFX10-NEXT: v_mov_b32_e32 v13, s11
+; GFX10-NEXT: v_mov_b32_e32 v12, s10
+; GFX10-NEXT: v_mov_b32_e32 v11, s9
+; GFX10-NEXT: v_mov_b32_e32 v10, s8
+; GFX10-NEXT: v_mov_b32_e32 v9, s7
+; GFX10-NEXT: v_mov_b32_e32 v8, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s4
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_movreld_b32_e32 v2, v0
; GFX10-NEXT: v_movreld_b32_e32 v3, v1
; GFX10-NEXT: v_readfirstlane_b32 s0, v2
@@ -7022,14 +7374,29 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_s(<7 x double> inreg
;
; GFX11-LABEL: dyn_insertelement_v7f64_s_v_s:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: s_mov_b32 s3, s5
+; GFX11-NEXT: s_mov_b32 s4, s6
+; GFX11-NEXT: s_mov_b32 s5, s7
+; GFX11-NEXT: s_mov_b32 s6, s8
+; GFX11-NEXT: s_mov_b32 s7, s9
+; GFX11-NEXT: s_mov_b32 s8, s10
+; GFX11-NEXT: s_mov_b32 s9, s11
+; GFX11-NEXT: s_mov_b32 s10, s12
+; GFX11-NEXT: s_mov_b32 s11, s13
+; GFX11-NEXT: s_mov_b32 s12, s14
+; GFX11-NEXT: s_mov_b32 s13, s15
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_lshl_b32 m0, s16, 1
-; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
-; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
-; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
-; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
-; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX11-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX11-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
+; GFX11-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX11-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX11-NEXT: v_movreld_b32_e32 v2, v0
; GFX11-NEXT: v_movreld_b32_e32 v3, v1
; GFX11-NEXT: v_readfirstlane_b32 s0, v2
@@ -7050,20 +7417,36 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_s(<7 x double> inreg
;
; GFX8-LABEL: dyn_insertelement_v7f64_s_v_s:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v7, s7
-; GFX8-NEXT: v_mov_b32_e32 v8, s8
-; GFX8-NEXT: v_mov_b32_e32 v9, s9
-; GFX8-NEXT: v_mov_b32_e32 v10, s10
-; GFX8-NEXT: v_mov_b32_e32 v11, s11
-; GFX8-NEXT: v_mov_b32_e32 v12, s12
-; GFX8-NEXT: v_mov_b32_e32 v13, s13
-; GFX8-NEXT: v_mov_b32_e32 v14, s14
-; GFX8-NEXT: v_mov_b32_e32 v15, s15
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: s_mov_b32 s4, s6
+; GFX8-NEXT: s_mov_b32 s5, s7
+; GFX8-NEXT: s_mov_b32 s6, s8
+; GFX8-NEXT: s_mov_b32 s7, s9
+; GFX8-NEXT: s_mov_b32 s8, s10
+; GFX8-NEXT: s_mov_b32 s9, s11
+; GFX8-NEXT: s_mov_b32 s10, s12
+; GFX8-NEXT: s_mov_b32 s11, s13
+; GFX8-NEXT: s_mov_b32 s12, s14
+; GFX8-NEXT: s_mov_b32 s13, s15
+; GFX8-NEXT: v_mov_b32_e32 v17, s15
+; GFX8-NEXT: v_mov_b32_e32 v16, s14
+; GFX8-NEXT: v_mov_b32_e32 v15, s13
+; GFX8-NEXT: v_mov_b32_e32 v14, s12
+; GFX8-NEXT: v_mov_b32_e32 v13, s11
+; GFX8-NEXT: v_mov_b32_e32 v12, s10
+; GFX8-NEXT: v_mov_b32_e32 v11, s9
+; GFX8-NEXT: v_mov_b32_e32 v10, s8
+; GFX8-NEXT: v_mov_b32_e32 v9, s7
+; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_lshl_b32 m0, s16, 1
; GFX8-NEXT: v_movreld_b32_e32 v2, v0
; GFX8-NEXT: v_movreld_b32_e32 v3, v1
@@ -7638,124 +8021,147 @@ entry:
define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_s_v_s(<5 x double> inreg %vec, double %val, i32 inreg %idx) {
; GPRIDX-LABEL: dyn_insertelement_v5f64_s_v_s:
; GPRIDX: ; %bb.0: ; %entry
-; GPRIDX-NEXT: s_cmp_eq_u32 s12, 4
-; GPRIDX-NEXT: v_mov_b32_e32 v2, s11
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: v_mov_b32_e32 v3, s10
-; GPRIDX-NEXT: s_cmp_eq_u32 s12, 3
-; GPRIDX-NEXT: v_cndmask_b32_e32 v2, v2, v1, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GPRIDX-NEXT: v_mov_b32_e32 v4, s9
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: v_mov_b32_e32 v5, s8
-; GPRIDX-NEXT: s_cmp_eq_u32 s12, 2
-; GPRIDX-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GPRIDX-NEXT: v_mov_b32_e32 v6, s7
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: v_mov_b32_e32 v7, s6
-; GPRIDX-NEXT: s_cmp_eq_u32 s12, 1
-; GPRIDX-NEXT: v_cndmask_b32_e32 v6, v6, v1, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GPRIDX-NEXT: v_mov_b32_e32 v8, s5
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: v_mov_b32_e32 v9, s4
-; GPRIDX-NEXT: s_cmp_eq_u32 s12, 0
-; GPRIDX-NEXT: v_cndmask_b32_e32 v8, v8, v1, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v9, v9, v0, vcc
-; GPRIDX-NEXT: v_mov_b32_e32 v10, s3
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc
-; GPRIDX-NEXT: v_mov_b32_e32 v10, s2
-; GPRIDX-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
-; GPRIDX-NEXT: v_readfirstlane_b32 s0, v0
-; GPRIDX-NEXT: v_readfirstlane_b32 s1, v1
-; GPRIDX-NEXT: v_readfirstlane_b32 s2, v9
-; GPRIDX-NEXT: v_readfirstlane_b32 s3, v8
-; GPRIDX-NEXT: v_readfirstlane_b32 s4, v7
-; GPRIDX-NEXT: v_readfirstlane_b32 s5, v6
-; GPRIDX-NEXT: v_readfirstlane_b32 s6, v5
-; GPRIDX-NEXT: v_readfirstlane_b32 s7, v4
-; GPRIDX-NEXT: v_readfirstlane_b32 s8, v3
-; GPRIDX-NEXT: v_readfirstlane_b32 s9, v2
+; GPRIDX-NEXT: s_mov_b32 s0, s2
+; GPRIDX-NEXT: s_mov_b32 s1, s3
+; GPRIDX-NEXT: s_mov_b32 s2, s4
+; GPRIDX-NEXT: s_mov_b32 s3, s5
+; GPRIDX-NEXT: s_mov_b32 s4, s6
+; GPRIDX-NEXT: s_mov_b32 s5, s7
+; GPRIDX-NEXT: s_mov_b32 s6, s8
+; GPRIDX-NEXT: s_mov_b32 s7, s9
+; GPRIDX-NEXT: s_mov_b32 s8, s10
+; GPRIDX-NEXT: s_mov_b32 s9, s11
+; GPRIDX-NEXT: v_mov_b32_e32 v11, s9
+; GPRIDX-NEXT: v_mov_b32_e32 v10, s8
+; GPRIDX-NEXT: v_mov_b32_e32 v9, s7
+; GPRIDX-NEXT: v_mov_b32_e32 v8, s6
+; GPRIDX-NEXT: v_mov_b32_e32 v7, s5
+; GPRIDX-NEXT: v_mov_b32_e32 v6, s4
+; GPRIDX-NEXT: v_mov_b32_e32 v5, s3
+; GPRIDX-NEXT: v_mov_b32_e32 v4, s2
+; GPRIDX-NEXT: v_mov_b32_e32 v3, s1
+; GPRIDX-NEXT: v_mov_b32_e32 v2, s0
+; GPRIDX-NEXT: s_lshl_b32 s0, s12, 1
+; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GPRIDX-NEXT: v_mov_b32_e32 v2, v0
+; GPRIDX-NEXT: v_mov_b32_e32 v3, v1
+; GPRIDX-NEXT: s_set_gpr_idx_off
+; GPRIDX-NEXT: v_readfirstlane_b32 s0, v2
+; GPRIDX-NEXT: v_readfirstlane_b32 s1, v3
+; GPRIDX-NEXT: v_readfirstlane_b32 s2, v4
+; GPRIDX-NEXT: v_readfirstlane_b32 s3, v5
+; GPRIDX-NEXT: v_readfirstlane_b32 s4, v6
+; GPRIDX-NEXT: v_readfirstlane_b32 s5, v7
+; GPRIDX-NEXT: v_readfirstlane_b32 s6, v8
+; GPRIDX-NEXT: v_readfirstlane_b32 s7, v9
+; GPRIDX-NEXT: v_readfirstlane_b32 s8, v10
+; GPRIDX-NEXT: v_readfirstlane_b32 s9, v11
; GPRIDX-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: dyn_insertelement_v5f64_s_v_s:
-; GFX10PLUS: ; %bb.0: ; %entry
-; GFX10PLUS-NEXT: s_cmp_eq_u32 s12, 4
-; GFX10PLUS-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10PLUS-NEXT: s_cmp_eq_u32 s12, 3
-; GFX10PLUS-NEXT: v_cndmask_b32_e32 v2, s11, v1, vcc_lo
-; GFX10PLUS-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10PLUS-NEXT: s_cmp_eq_u32 s12, 2
-; GFX10PLUS-NEXT: v_cndmask_b32_e32 v3, s10, v0, vcc_lo
-; GFX10PLUS-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10PLUS-NEXT: s_cmp_eq_u32 s12, 1
-; GFX10PLUS-NEXT: v_cndmask_b32_e64 v4, s9, v1, s0
-; GFX10PLUS-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10PLUS-NEXT: s_cmp_eq_u32 s12, 0
-; GFX10PLUS-NEXT: v_cndmask_b32_e64 v5, s8, v0, s0
-; GFX10PLUS-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10PLUS-NEXT: v_cndmask_b32_e64 v8, s5, v1, s1
-; GFX10PLUS-NEXT: v_cndmask_b32_e64 v6, s3, v1, s0
-; GFX10PLUS-NEXT: v_cndmask_b32_e64 v7, s2, v0, s0
-; GFX10PLUS-NEXT: v_cndmask_b32_e64 v9, s4, v0, s1
-; GFX10PLUS-NEXT: v_cndmask_b32_e32 v1, s7, v1, vcc_lo
-; GFX10PLUS-NEXT: v_cndmask_b32_e32 v0, s6, v0, vcc_lo
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v6
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v7
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v9
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v8
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s6, v5
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s7, v4
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s8, v3
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: dyn_insertelement_v5f64_s_v_s:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: s_mov_b32 s8, s10
+; GFX10-NEXT: s_mov_b32 s9, s11
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: s_lshl_b32 m0, s12, 1
+; GFX10-NEXT: v_mov_b32_e32 v11, s9
+; GFX10-NEXT: v_mov_b32_e32 v10, s8
+; GFX10-NEXT: v_mov_b32_e32 v9, s7
+; GFX10-NEXT: v_mov_b32_e32 v8, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s4
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_movreld_b32_e32 v2, v0
+; GFX10-NEXT: v_movreld_b32_e32 v3, v1
+; GFX10-NEXT: v_readfirstlane_b32 s0, v2
+; GFX10-NEXT: v_readfirstlane_b32 s1, v3
+; GFX10-NEXT: v_readfirstlane_b32 s2, v4
+; GFX10-NEXT: v_readfirstlane_b32 s3, v5
+; GFX10-NEXT: v_readfirstlane_b32 s4, v6
+; GFX10-NEXT: v_readfirstlane_b32 s5, v7
+; GFX10-NEXT: v_readfirstlane_b32 s6, v8
+; GFX10-NEXT: v_readfirstlane_b32 s7, v9
+; GFX10-NEXT: v_readfirstlane_b32 s8, v10
+; GFX10-NEXT: v_readfirstlane_b32 s9, v11
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: dyn_insertelement_v5f64_s_v_s:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: s_mov_b32 s3, s5
+; GFX11-NEXT: s_mov_b32 s4, s6
+; GFX11-NEXT: s_mov_b32 s5, s7
+; GFX11-NEXT: s_mov_b32 s6, s8
+; GFX11-NEXT: s_mov_b32 s7, s9
+; GFX11-NEXT: s_mov_b32 s8, s10
+; GFX11-NEXT: s_mov_b32 s9, s11
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_lshl_b32 m0, s12, 1
+; GFX11-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
+; GFX11-NEXT: v_movreld_b32_e32 v2, v0
+; GFX11-NEXT: v_movreld_b32_e32 v3, v1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: v_readfirstlane_b32 s1, v3
+; GFX11-NEXT: v_readfirstlane_b32 s2, v4
+; GFX11-NEXT: v_readfirstlane_b32 s3, v5
+; GFX11-NEXT: v_readfirstlane_b32 s4, v6
+; GFX11-NEXT: v_readfirstlane_b32 s5, v7
+; GFX11-NEXT: v_readfirstlane_b32 s6, v8
+; GFX11-NEXT: v_readfirstlane_b32 s7, v9
+; GFX11-NEXT: v_readfirstlane_b32 s8, v10
+; GFX11-NEXT: v_readfirstlane_b32 s9, v11
+; GFX11-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: dyn_insertelement_v5f64_s_v_s:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_cmp_eq_u32 s12, 4
-; GFX8-NEXT: v_mov_b32_e32 v2, s11
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v3, s10
-; GFX8-NEXT: s_cmp_eq_u32 s12, 3
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s9
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v5, s8
-; GFX8-NEXT: s_cmp_eq_u32 s12, 2
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s7
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v7, s6
-; GFX8-NEXT: s_cmp_eq_u32 s12, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v8, s5
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v9, s4
-; GFX8-NEXT: s_cmp_eq_u32 s12, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v10, s3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v10, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s2, v9
-; GFX8-NEXT: v_readfirstlane_b32 s3, v8
-; GFX8-NEXT: v_readfirstlane_b32 s4, v7
-; GFX8-NEXT: v_readfirstlane_b32 s5, v6
-; GFX8-NEXT: v_readfirstlane_b32 s6, v5
-; GFX8-NEXT: v_readfirstlane_b32 s7, v4
-; GFX8-NEXT: v_readfirstlane_b32 s8, v3
-; GFX8-NEXT: v_readfirstlane_b32 s9, v2
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: s_mov_b32 s4, s6
+; GFX8-NEXT: s_mov_b32 s5, s7
+; GFX8-NEXT: s_mov_b32 s6, s8
+; GFX8-NEXT: s_mov_b32 s7, s9
+; GFX8-NEXT: s_mov_b32 s8, s10
+; GFX8-NEXT: s_mov_b32 s9, s11
+; GFX8-NEXT: v_mov_b32_e32 v11, s9
+; GFX8-NEXT: v_mov_b32_e32 v10, s8
+; GFX8-NEXT: v_mov_b32_e32 v9, s7
+; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: s_lshl_b32 m0, s12, 1
+; GFX8-NEXT: v_movreld_b32_e32 v2, v0
+; GFX8-NEXT: v_movreld_b32_e32 v3, v1
+; GFX8-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s1, v3
+; GFX8-NEXT: v_readfirstlane_b32 s2, v4
+; GFX8-NEXT: v_readfirstlane_b32 s3, v5
+; GFX8-NEXT: v_readfirstlane_b32 s4, v6
+; GFX8-NEXT: v_readfirstlane_b32 s5, v7
+; GFX8-NEXT: v_readfirstlane_b32 s6, v8
+; GFX8-NEXT: v_readfirstlane_b32 s7, v9
+; GFX8-NEXT: v_readfirstlane_b32 s8, v10
+; GFX8-NEXT: v_readfirstlane_b32 s9, v11
; GFX8-NEXT: ; return to shader part epilog
entry:
%insert = insertelement <5 x double> %vec, double %val, i32 %idx
@@ -7959,26 +8365,11 @@ entry:
define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_s(<5 x double> %vec, double %val, i32 inreg %idx) {
; GPRIDX-LABEL: dyn_insertelement_v5f64_v_v_s:
; GPRIDX: ; %bb.0: ; %entry
-; GPRIDX-NEXT: s_cmp_eq_u32 s2, 4
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: s_cmp_eq_u32 s2, 3
-; GPRIDX-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: s_cmp_eq_u32 s2, 2
-; GPRIDX-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: s_cmp_eq_u32 s2, 1
-; GPRIDX-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: s_cmp_eq_u32 s2, 0
-; GPRIDX-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
-; GPRIDX-NEXT: s_cselect_b64 vcc, -1, 0
-; GPRIDX-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
+; GPRIDX-NEXT: s_lshl_b32 s0, s2, 1
+; GPRIDX-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GPRIDX-NEXT: v_mov_b32_e32 v0, v10
+; GPRIDX-NEXT: v_mov_b32_e32 v1, v11
+; GPRIDX-NEXT: s_set_gpr_idx_off
; GPRIDX-NEXT: v_readfirstlane_b32 s0, v0
; GPRIDX-NEXT: v_readfirstlane_b32 s1, v1
; GPRIDX-NEXT: v_readfirstlane_b32 s2, v2
@@ -7991,94 +8382,28 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_s(<5 x double> %vec,
; GPRIDX-NEXT: v_readfirstlane_b32 s9, v9
; GPRIDX-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: dyn_insertelement_v5f64_v_v_s:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_cmp_eq_u32 s2, 4
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 3
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc_lo
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 2
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 1
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v11, s0
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v10, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v11, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v10, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v10, s1
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc_lo
-; GFX10-NEXT: v_readfirstlane_b32 s1, v1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s2, v2
-; GFX10-NEXT: v_readfirstlane_b32 s3, v3
-; GFX10-NEXT: v_readfirstlane_b32 s4, v4
-; GFX10-NEXT: v_readfirstlane_b32 s5, v5
-; GFX10-NEXT: v_readfirstlane_b32 s6, v6
-; GFX10-NEXT: v_readfirstlane_b32 s7, v7
-; GFX10-NEXT: v_readfirstlane_b32 s8, v8
-; GFX10-NEXT: v_readfirstlane_b32 s9, v9
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: dyn_insertelement_v5f64_v_v_s:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_cmp_eq_u32 s2, 4
-; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 3
-; GFX11-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_cndmask_b32 v8, v8, v10
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 2
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, v11, s0
-; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 1
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v10, s0
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v11, s1
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v10, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v11, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v10, s0
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v11 :: v_dual_cndmask_b32 v4, v4, v10
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: v_readfirstlane_b32 s4, v4
-; GFX11-NEXT: v_readfirstlane_b32 s5, v5
-; GFX11-NEXT: v_readfirstlane_b32 s6, v6
-; GFX11-NEXT: v_readfirstlane_b32 s7, v7
-; GFX11-NEXT: v_readfirstlane_b32 s8, v8
-; GFX11-NEXT: v_readfirstlane_b32 s9, v9
-; GFX11-NEXT: ; return to shader part epilog
+; GFX10PLUS-LABEL: dyn_insertelement_v5f64_v_v_s:
+; GFX10PLUS: ; %bb.0: ; %entry
+; GFX10PLUS-NEXT: s_lshl_b32 m0, s2, 1
+; GFX10PLUS-NEXT: v_movreld_b32_e32 v0, v10
+; GFX10PLUS-NEXT: v_movreld_b32_e32 v1, v11
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s4, v4
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s5, v5
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s6, v6
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s7, v7
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s8, v8
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s9, v9
+; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: dyn_insertelement_v5f64_v_v_s:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_cmp_eq_u32 s2, 4
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 3
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 2
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX8-NEXT: s_lshl_b32 m0, s2, 1
+; GFX8-NEXT: v_movreld_b32_e32 v0, v10
+; GFX8-NEXT: v_movreld_b32_e32 v1, v11
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: v_readfirstlane_b32 s1, v1
; GFX8-NEXT: v_readfirstlane_b32 s2, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
index 99202ff7fe326..952eab93ced18 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=tahiti -o - < %s | FileCheck %s --check-prefixes=GFX,GFX6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=fiji -o - < %s | FileCheck %s --check-prefixes=GFX,GFX8
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX10
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX10
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-FAKE16
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-REAL16
@@ -175,39 +175,43 @@ define i64 @abs_vgpr_i64(i64 %arg) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX6-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX6-NEXT: v_xor_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: abs_vgpr_i64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX8-NEXT: v_xor_b32_e32 v1, v1, v2
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: abs_vgpr_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo
; GFX10-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX10-NEXT: v_xor_b32_e32 v1, v1, v2
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: abs_vgpr_i64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_sub_nc_u64_e32 v[2:3], 0, v[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_max_i64 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mov_b32_e32 v3, v2
+; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX1250-NEXT: v_xor_b32_e32 v1, v1, v2
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%res = call i64 @llvm.abs.i64(i64 %arg, i1 false)
ret i64 %res
@@ -483,18 +487,22 @@ define amdgpu_cs <2 x i16> @abs_sgpr_v2i16(<2 x i16> inreg %arg) {
;
; GFX10-LABEL: abs_sgpr_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_sub_i16 v0, 0, s0
-; GFX10-NEXT: v_pk_max_i16 v0, s0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_sext_i32_i16 s1, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10-NEXT: s_abs_i32 s1, s1
+; GFX10-NEXT: s_abs_i32 s0, s0
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: abs_sgpr_v2i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_pk_sub_i16 v0, 0, s0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_i16 v0, s0, v0
-; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_sext_i32_i16 s1, s0
+; GFX1250-NEXT: s_ashr_i32 s0, s0, 16
+; GFX1250-NEXT: s_abs_i32 s1, s1
+; GFX1250-NEXT: s_abs_i32 s0, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX1250-NEXT: ; return to shader part epilog
%res = call <2 x i16> @llvm.abs.v2i16(<2 x i16> %arg, i1 false)
ret <2 x i16> %res
@@ -571,25 +579,25 @@ define amdgpu_cs <3 x i16> @abs_sgpr_v3i16(<3 x i16> inreg %arg) {
;
; GFX10-LABEL: abs_sgpr_v3i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_sub_i16 v0, 0, s0
-; GFX10-NEXT: v_pk_sub_i16 v1, 0, s1
-; GFX10-NEXT: v_pk_max_i16 v0, s0, v0
-; GFX10-NEXT: v_pk_max_i16 v1, s1, v1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_sext_i32_i16 s2, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10-NEXT: s_abs_i32 s2, s2
+; GFX10-NEXT: s_abs_i32 s0, s0
+; GFX10-NEXT: s_sext_i32_i16 s1, s1
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX10-NEXT: s_abs_i32 s1, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: abs_sgpr_v3i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_pk_sub_i16 v0, 0, s0
-; GFX1250-NEXT: v_pk_sub_i16 v1, 0, s1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_pk_max_i16 v0, s0, v0
-; GFX1250-NEXT: v_pk_max_i16 v1, s1, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: s_sext_i32_i16 s2, s0
+; GFX1250-NEXT: s_ashr_i32 s0, s0, 16
+; GFX1250-NEXT: s_abs_i32 s2, s2
+; GFX1250-NEXT: s_abs_i32 s0, s0
+; GFX1250-NEXT: s_sext_i32_i16 s1, s1
+; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX1250-NEXT: s_abs_i32 s1, s1
; GFX1250-NEXT: ; return to shader part epilog
%res = call <3 x i16> @llvm.abs.v3i16(<3 x i16> %arg, i1 false)
ret <3 x i16> %res
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
index 72253efefbc7f..bdb342f7166aa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
@@ -358,15 +358,14 @@ main_body:
define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #0 {
; GFX11-TRUE16-LABEL: v_interp_f16_imm_params:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_interp_p10_f16_f32 v1, v0.l, v1, v0.l wait_exp:7
+; GFX11-TRUE16-NEXT: v_interp_p2_f16_f32 v0.l, v0.l, v2, v3 wait_exp:7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_interp_p10_f16_f32 v2, v0.l, v2, v0.l wait_exp:7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_interp_p2_f16_f32 v0.l, v0.l, v3, v1 wait_exp:7
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, v1
; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
index 10db379f02479..8c5a01c80b0cd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
@@ -415,15 +415,15 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -458,20 +458,20 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(i64) = G_PTRTOINT [[MV]](p0)
; CHECK45-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 25
; CHECK45-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; CHECK45-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[COPY3]], [[COPY6]](s32)
- ; CHECK45-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[PTRTOINT]](i64)
+ ; CHECK45-NEXT: [[SHL:%[0-9]+]]:vgpr(i32) = G_SHL [[COPY3]], [[COPY6]](s32)
+ ; CHECK45-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[PTRTOINT]](i64)
; CHECK45-NEXT: [[OR:%[0-9]+]]:vgpr(s32) = G_OR [[UV1]], [[SHL]]
; CHECK45-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 7
; CHECK45-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C2]](i32)
; CHECK45-NEXT: [[LSHR:%[0-9]+]]:vgpr(i64) = G_LSHR [[MV1]], [[COPY7]](i32)
; CHECK45-NEXT: [[C3:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 12
; CHECK45-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C3]](i32)
- ; CHECK45-NEXT: [[SHL1:%[0-9]+]]:vgpr(s32) = G_SHL [[COPY2]], [[COPY8]](i32)
+ ; CHECK45-NEXT: [[SHL1:%[0-9]+]]:vgpr(i32) = G_SHL [[COPY2]], [[COPY8]](i32)
; CHECK45-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 28
; CHECK45-NEXT: [[COPY9:%[0-9]+]]:vgpr(i32) = COPY [[C4]](i32)
- ; CHECK45-NEXT: [[SHL2:%[0-9]+]]:vgpr(s32) = G_SHL [[COPY5]], [[COPY9]](i32)
- ; CHECK45-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[LSHR]](i64)
- ; CHECK45-NEXT: [[OR1:%[0-9]+]]:vgpr(s32) = G_OR [[UV3]], [[SHL1]]
+ ; CHECK45-NEXT: [[SHL2:%[0-9]+]]:vgpr(i32) = G_SHL [[COPY5]], [[COPY9]](i32)
+ ; CHECK45-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[LSHR]](i64)
+ ; CHECK45-NEXT: [[OR1:%[0-9]+]]:vgpr(i32) = G_OR [[UV3]], [[SHL1]]
; CHECK45-NEXT: [[OR2:%[0-9]+]]:vgpr(s32) = G_OR [[OR1]], [[SHL2]]
; CHECK45-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128_align2(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[OR]](s32), [[UV2]](s32), [[OR2]](s32)
; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[C]](i32)
@@ -487,15 +487,15 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK45-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[OR2]](s32)
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK45-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK45-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK45-NEXT: [[COPY15:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
- ; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](s64), [[COPY13]](s64), implicit $exec
+ ; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK45-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK45-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK45-NEXT: [[COPY15:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
+ ; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](i64), [[COPY13]](i64), implicit $exec
; CHECK45-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK45-NEXT: [[COPY16:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; CHECK45-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY16]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK45-NEXT: [[COPY16:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; CHECK45-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY16]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK45-NEXT: {{ $}}
; CHECK45-NEXT: bb.3:
; CHECK45-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index f3779b9b48c35..a88f059168b94 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -268,15 +268,15 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY2]](i32), implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY3]](i32), implicit $exec
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[V_READFIRSTLANE_B32_]](i32), %subreg.sub0, [[V_READFIRSTLANE_B32_1]](i32), %subreg.sub1, [[V_READFIRSTLANE_B32_2]](i32), %subreg.sub2, [[V_READFIRSTLANE_B32_3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x i32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x i32>)
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -711,15 +711,15 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY2]](i32), implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY3]](i32), implicit $exec
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[V_READFIRSTLANE_B32_]](i32), %subreg.sub0, [[V_READFIRSTLANE_B32_1]](i32), %subreg.sub1, [[V_READFIRSTLANE_B32_2]](i32), %subreg.sub2, [[V_READFIRSTLANE_B32_3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x i32>)
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x i32>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.add.ll
index f5cbcbd432872..f555c9e9f6cc1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.add.ll
@@ -119,18 +119,18 @@ define amdgpu_ps float @raw_ptr_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -183,18 +183,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc_
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.ll
index 35f2f6f134c10..5b1d8a8b718dd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.ll
@@ -80,18 +80,18 @@ define amdgpu_ps float @raw_ptr_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__v
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](s64), [[COPY13]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](i64), [[COPY13]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -148,18 +148,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_c
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](s64), [[COPY13]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](i64), [[COPY13]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -301,18 +301,18 @@ define amdgpu_ps double @raw_ptr_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](s64), [[COPY14]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](s64), [[COPY15]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](i64), [[COPY14]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](i64), [[COPY15]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY9]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY9]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY18]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY18]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -378,18 +378,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_c
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](s64), [[COPY14]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](s64), [[COPY15]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](i64), [[COPY14]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](i64), [[COPY15]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY9]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY9]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY18]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY18]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.ll
index 98dc3f26d9222..6bacd2f704f2a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.ll
@@ -167,18 +167,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc_
; GFX908-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX908-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX908-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; GFX908-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX908-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; GFX908-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX908-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX908-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3:
; GFX908-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -223,18 +223,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc_
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX90A-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX90A-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3:
; GFX90A-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -283,18 +283,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc_
; GFX908-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX908-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX908-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX908-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; GFX908-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; GFX908-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX908-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; GFX908-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3:
; GFX908-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -338,18 +338,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc_
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX90A-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX90A-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; GFX90A-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX90A-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3:
; GFX90A-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.sub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.sub.ll
index 8e75f8fd56de6..dd4e1c0751d42 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.sub.ll
@@ -119,18 +119,18 @@ define amdgpu_ps float @raw_ptr_buffer_atomic_sub_i32__sgpr_val__vgpr_rsrc__sgpr
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -183,18 +183,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_sub_i32_noret__sgpr_val__vgpr_rsrc_
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.swap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.swap.ll
index 26afeb8c47e20..10d1a2d25013a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.swap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.swap.ll
@@ -116,18 +116,18 @@ define amdgpu_ps float @raw_ptr_buffer_atomic_swap_i32__sgpr_val__vgpr_rsrc__sgp
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -179,18 +179,18 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_swap_i32_noret__sgpr_val__vgpr_rsrc
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll
index 5be6a73765d7d..f3cdd51416c08 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll
@@ -177,18 +177,18 @@ define amdgpu_ps half @raw_ptr_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -232,18 +232,18 @@ define amdgpu_ps half @raw_ptr_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.ll
index c2aa6e01ccca5..5b0da6983181d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.ll
@@ -121,18 +121,18 @@ define amdgpu_ps float @raw_ptr_buffer_load_format_f32__vgpr_rsrc__sgpr_voffset_
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
index 2582a00790b14..8c47a921916a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -102,15 +102,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -153,15 +153,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_s
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -209,18 +209,18 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_s
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -263,18 +263,18 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_s
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; GFX1250-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; GFX1250-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[S_AND_B32_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_1]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_1]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -957,15 +957,15 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_so
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -1008,15 +1008,15 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_so
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -1064,15 +1064,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_so
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -1116,15 +1116,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_so
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -1612,15 +1612,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY11]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -1665,15 +1665,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](s64), [[COPY7]](s64), implicit $exec
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY9]](i64), [[COPY7]](i64), implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY11]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -1725,15 +1725,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -1779,15 +1779,15 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; GFX1250-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
index 4c582b6a963e9..d9d46c06990e9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
@@ -213,15 +213,15 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -266,15 +266,15 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -539,15 +539,15 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](s64), [[COPY13]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](i64), [[COPY13]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -595,15 +595,15 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f32.ll
index fb8136ccbd122..c7d8c0e88e98a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f32.ll
@@ -136,15 +136,15 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -311,15 +311,15 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](s64), [[COPY13]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](i64), [[COPY13]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY16]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll
index 6f9c292f73ce8..d26ef50a36172 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll
@@ -72,15 +72,15 @@ define amdgpu_ps void @raw_ptr_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -171,18 +171,18 @@ define amdgpu_ps void @raw_ptr_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -554,15 +554,15 @@ define amdgpu_ps void @raw_ptr_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -815,15 +815,15 @@ define amdgpu_ps void @raw_ptr_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -873,15 +873,15 @@ define amdgpu_ps void @raw_ptr_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll
index d562b9a8cba41..8c438125e9592 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll
@@ -175,18 +175,18 @@ define amdgpu_ps half @raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -230,18 +230,18 @@ define amdgpu_ps half @raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.ll
index 3eeb7e68dd570..6bd6e3afb6ec0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.ll
@@ -120,18 +120,18 @@ define amdgpu_ps float @raw_tbuffer_load_f32__vgpr_rsrc__sgpr_voffset__vgpr_soff
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY5]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY5]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[S_AND_B32_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.i8.ll
index 4257722d0696a..3c2de37593435 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.i8.ll
@@ -64,15 +64,15 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -115,15 +115,15 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -171,18 +171,18 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -225,18 +225,18 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -285,18 +285,18 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -340,18 +340,18 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.ll
index f1959f327778e..e76d887580d71 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.ll
@@ -140,15 +140,15 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__sgpr_soff
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -196,18 +196,18 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__vgpr_soff
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[S_AND_B32_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -256,18 +256,18 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__sgpr_voffset__vgpr_soff
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[S_AND_B32_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY13]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY13]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -622,15 +622,15 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](s64), [[COPY8]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY12]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -682,15 +682,15 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(s32) = COPY [[S_AND_B32_]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY13]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(i32) = COPY [[S_AND_B32_]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[COPY13]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll
index e2392cc6ead17..68926df513552 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll
@@ -1,6 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; UNSUPPORTED: true
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=GFX6 %s
define i32 @v_bfe_i32_arg_arg_arg(i32 %src0, i32 %src1, i32 %src2) #0 {
; GFX6-LABEL: v_bfe_i32_arg_arg_arg:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
index 37c9339344f4b..d4c23d63db7c0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
@@ -237,16 +237,16 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__vpr_rsrc__sgpr_vin
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](i32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](i32)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(s64), [[UV3:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](s64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](s64), [[UV1]]
+ ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
+ ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.add.ll
index f73afc0241571..6fafecd3d1b45 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.add.ll
@@ -127,18 +127,18 @@ define amdgpu_ps float @struct_ptr_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__s
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -192,18 +192,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rs
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.ll
index ca63c9176c967..1af2548c28f05 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.ll
@@ -84,18 +84,18 @@ define amdgpu_ps float @struct_ptr_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](s64), [[COPY13]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](s64), [[COPY14]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](i64), [[COPY13]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](i64), [[COPY14]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY8]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY8]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY17]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY17]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -153,18 +153,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgp
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](s64), [[COPY13]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](s64), [[COPY14]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY15]](i64), [[COPY13]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](i64), [[COPY14]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY8]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY8]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY17]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY17]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -313,18 +313,18 @@ define amdgpu_ps double @struct_ptr_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cm
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](s64), [[COPY15]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY18]](s64), [[COPY16]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](i64), [[COPY15]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY18]](i64), [[COPY16]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY10]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY10]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY19:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY19]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY19:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY19]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -391,18 +391,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgp
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](s64), [[COPY15]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY18]](s64), [[COPY16]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](i64), [[COPY15]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY18]](i64), [[COPY16]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY10]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY10]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY19:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY19]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY19:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY19]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.ll
index 4715097e0f99d..3ace08a4333d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.ll
@@ -179,18 +179,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rs
; GFX908-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX908-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX908-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; GFX908-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX908-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; GFX908-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX908-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX908-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3:
; GFX908-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -236,18 +236,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rs
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX90A-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX90A-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX90A-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3:
; GFX90A-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -298,18 +298,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rs
; GFX908-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX908-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX908-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX908-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; GFX908-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX908-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; GFX908-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX908-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX908-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3:
; GFX908-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -354,18 +354,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rs
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; GFX90A-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; GFX90A-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
- ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
+ ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; GFX90A-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; GFX90A-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; GFX90A-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; GFX90A-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3:
; GFX90A-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.sub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.sub.ll
index fa5fb3abca696..3955a9fda5d3c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.sub.ll
@@ -127,18 +127,18 @@ define amdgpu_ps float @struct_ptr_buffer_atomic_sub_i32__sgpr_val__vgpr_rsrc__s
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -192,18 +192,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_sub_i32_noret__sgpr_val__vgpr_rs
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.swap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.swap.ll
index 384be118e4222..ddbb12cecf56e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.swap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.swap.ll
@@ -124,18 +124,18 @@ define amdgpu_ps float @struct_ptr_buffer_atomic_swap_i32__sgpr_val__vgpr_rsrc__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -188,18 +188,18 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_swap_i32_noret__sgpr_val__vgpr_r
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
index b6fe7adabd123..08c7e61d7a671 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
@@ -182,16 +182,16 @@ define amdgpu_ps <4 x half> @struct_ptr_buffer_load_format_v4f16__vpr_rsrc__sgpr
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(s64), [[UV3:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](s64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](s64), [[UV1]]
+ ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
+ ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -255,18 +255,18 @@ define amdgpu_ps <4 x half> @struct_ptr_buffer_load_format_v4f16__vpr_rsrc__sgpr
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
index 26a2670877d21..dc90b8b9bd06b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
@@ -129,18 +129,18 @@ define amdgpu_ps <4 x float> @struct_ptr_buffer_load_format_v4f32__vpr_rsrc__sgp
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.ll
index d8cee55cdf589..d51be42feb870 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.ll
@@ -199,18 +199,18 @@ define amdgpu_ps float @struct_ptr_buffer_load_f32__vgpr_rsrc__sgpr_vindex__sgpr
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f32.ll
index 8dafbd8bf2d60..e1009a7723657 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f32.ll
@@ -123,18 +123,18 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_f32__sgpr_val__vgpr_rsrc__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](s64), [[COPY11]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](s64), [[COPY12]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.ll
index b34e22b1f9e60..11a8399171d0c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.ll
@@ -128,18 +128,18 @@ define amdgpu_ps void @struct_ptr_buffer_store_v4f32_vgpr_rsrc__sgpr_val__sgpr_v
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](s64), [[COPY14]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](s64), [[COPY15]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR1]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY16]](i64), [[COPY14]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY17]](i64), [[COPY15]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY10]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY10]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY18]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY18]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
index 080e34a14b3f5..92fd5ddf73817 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
@@ -223,18 +223,18 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -282,16 +282,16 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(s64), [[UV3:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](s64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](s64), [[UV1]]
+ ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
+ ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.ll
index 583a2016d0e28..395862ba7a903 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.ll
@@ -151,18 +151,18 @@ define amdgpu_ps <4 x float> @struct_tbuffer_load_v4f32__vgpr_rsrc__sgpr_vindex_
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(s64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(s64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](s64), [[COPY9]](s64), implicit $exec
- ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](s64), [[COPY10]](s64), implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; CHECK-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; CHECK-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(s64) = COPY [[S_AND_B64_1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
index a8b856124ca2e..61abb85fa671e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
@@ -410,16 +410,16 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](i32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](i32)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(s64), [[UV3:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](s64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](s64), [[UV1]]
+ ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
+ ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
index a3e49834aa1df..95e90d116778c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
@@ -1,6 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; UNSUPPORTED: true
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=GFX6 %s
define i32 @v_bfe_i32_arg_arg_arg(i32 %src0, i32 %src1, i32 %src2) #0 {
; GFX6-LABEL: v_bfe_i32_arg_arg_arg:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
index a4b3f429b1de6..29d6615657d64 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
@@ -30,14 +30,14 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx12(ptr addrspace(1) inreg %ptr
;
; GFX11-True16-LABEL: load_uniform_P1_i16_b16_gfx12:
; GFX11-True16: ; %bb.0:
-; GFX11-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-True16-NEXT: s_clause 0x1
-; GFX11-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX11-True16-NEXT: global_load_d16_b16 v3, v3, s[2:3] glc dlc
+; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[2:3] glc dlc
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-True16-NEXT: s_add_u32 s0, s0, s1
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -60,14 +60,14 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx12(ptr addrspace(1) inreg %ptr
;
; GFX12-True16-LABEL: load_uniform_P1_i16_b16_gfx12:
; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX12-True16-NEXT: s_clause 0x1
-; GFX12-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX12-True16-NEXT: global_load_d16_b16 v3, v3, s[2:3] scope:SCOPE_SYS
+; GFX12-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX12-True16-NEXT: global_load_d16_b16 v2, v2, s[2:3] scope:SCOPE_SYS
; GFX12-True16-NEXT: s_wait_loadcnt 0x0
-; GFX12-True16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX12-True16-NEXT: s_add_co_u32 s0, s0, s1
+; GFX12-True16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX12-True16-NEXT: s_add_co_i32 s0, s0, s1
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -114,14 +114,14 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx11(ptr addrspace(1) inreg %ptr
;
; GFX11-True16-LABEL: load_uniform_P1_i16_b16_gfx11:
; GFX11-True16: ; %bb.0:
-; GFX11-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-True16-NEXT: s_clause 0x1
-; GFX11-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX11-True16-NEXT: global_load_d16_b16 v3, v3, s[0:1] glc dlc
+; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[0:1] glc dlc
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-True16-NEXT: s_add_u32 s0, s0, s1
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -150,7 +150,7 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx11(ptr addrspace(1) inreg %ptr
; GFX12-True16-NEXT: s_wait_loadcnt 0x0
; GFX12-True16-NEXT: v_readfirstlane_b32 s0, v2
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_u32 s0, s2, s0
+; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -196,14 +196,14 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx12(ptr addrspace(1) i
;
; GFX11-True16-LABEL: load_uniform_P1_i16_anyextending_gfx12:
; GFX11-True16: ; %bb.0:
-; GFX11-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-True16-NEXT: s_clause 0x1
-; GFX11-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX11-True16-NEXT: global_load_d16_b16 v3, v3, s[0:1] glc dlc
+; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[0:1] glc dlc
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-True16-NEXT: s_add_u32 s0, s0, s1
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -226,14 +226,14 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx12(ptr addrspace(1) i
;
; GFX12-True16-LABEL: load_uniform_P1_i16_anyextending_gfx12:
; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX12-True16-NEXT: s_clause 0x1
-; GFX12-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX12-True16-NEXT: global_load_d16_b16 v3, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX12-True16-NEXT: global_load_d16_b16 v2, v2, s[0:1] scope:SCOPE_SYS
; GFX12-True16-NEXT: s_wait_loadcnt 0x0
-; GFX12-True16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX12-True16-NEXT: s_add_co_u32 s0, s0, s1
+; GFX12-True16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX12-True16-NEXT: s_add_co_i32 s0, s0, s1
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -280,14 +280,14 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx11(ptr addrspace(1) i
;
; GFX11-True16-LABEL: load_uniform_P1_i16_anyextending_gfx11:
; GFX11-True16: ; %bb.0:
-; GFX11-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-True16-NEXT: s_clause 0x1
-; GFX11-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX11-True16-NEXT: global_load_d16_b16 v3, v3, s[0:1] glc dlc
+; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[0:1] glc dlc
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-True16-NEXT: s_add_u32 s0, s0, s1
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -316,7 +316,7 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx11(ptr addrspace(1) i
; GFX12-True16-NEXT: s_wait_loadcnt 0x0
; GFX12-True16-NEXT: v_readfirstlane_b32 s0, v2
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_u32 s0, s2, s0
+; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1250,15 +1250,15 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
;
; GFX11-True16-LABEL: load_uniform_P4_i16_b16_gfx12:
; GFX11-True16: ; %bb.0:
-; GFX11-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-True16-NEXT: s_clause 0x1
-; GFX11-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX11-True16-NEXT: global_load_d16_b16 v3, v3, s[2:3] glc dlc
+; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[2:3] glc dlc
; GFX11-True16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-True16-NEXT: s_add_u32 s0, s0, s1
+; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1288,7 +1288,7 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX12-True16-NEXT: s_wait_loadcnt 0x0
; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_u32 s0, s1, s0
+; GFX12-True16-NEXT: s_add_co_i32 s0, s1, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1340,7 +1340,7 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx11(ptr addrspace(4) inreg %ptr
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-True16-NEXT: s_add_u32 s0, s1, s0
+; GFX11-True16-NEXT: s_add_i32 s0, s1, s0
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1366,7 +1366,7 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx11(ptr addrspace(4) inreg %ptr
; GFX12-True16-NEXT: s_load_u16 s2, s[0:1], 0x0
; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_u32 s0, s2, s0
+; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1411,15 +1411,15 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
;
; GFX11-True16-LABEL: load_uniform_P4_i16_anyextending_gfx12:
; GFX11-True16: ; %bb.0:
-; GFX11-True16-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-True16-NEXT: s_clause 0x1
-; GFX11-True16-NEXT: global_load_d16_b16 v2, v3, s[0:1]
-; GFX11-True16-NEXT: global_load_d16_b16 v3, v3, s[0:1] glc dlc
+; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
+; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[0:1] glc dlc
; GFX11-True16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-True16-NEXT: s_add_u32 s0, s0, s1
+; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1449,7 +1449,7 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX12-True16-NEXT: s_wait_loadcnt 0x0
; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_u32 s0, s1, s0
+; GFX12-True16-NEXT: s_add_co_i32 s0, s1, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1501,7 +1501,7 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx11(ptr addrspace(4) i
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-True16-NEXT: s_add_u32 s0, s1, s0
+; GFX11-True16-NEXT: s_add_i32 s0, s1, s0
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
@@ -1527,7 +1527,7 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx11(ptr addrspace(4) i
; GFX12-True16-NEXT: s_load_u16 s2, s[0:1], 0x0
; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_u32 s0, s2, s0
+; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index 0c8039aeee6df..6afa4c8b7cd8f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; xUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
@@ -79,7 +79,7 @@ define amdgpu_ps i8 @s_lshr_i8(i8 inreg %value, i8 inreg %amount) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s0, 0xff
; GFX8-NEXT: s_and_b32 s1, s1, 0xff
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshr_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
@@ -87,7 +87,7 @@ define amdgpu_ps i8 @s_lshr_i8(i8 inreg %value, i8 inreg %amount) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s0, 0xff
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
; GFX9-NEXT: s_lshr_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -95,7 +95,7 @@ define amdgpu_ps i8 @s_lshr_i8(i8 inreg %value, i8 inreg %amount) {
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10PLUS-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX10PLUS-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, s1
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr i8 %value, %amount
@@ -111,21 +111,21 @@ define amdgpu_ps i8 @s_lshr_i8_7(i8 inreg %value) {
; GFX8-LABEL: s_lshr_i8_7:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshr_b32 s0, s0, 7
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_lshr_i8_7:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
; GFX9-NEXT: s_lshr_b32 s0, s0, 7
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_lshr_i8_7:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX10PLUS-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, 7
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr i8 %value, 7
@@ -661,19 +661,19 @@ define amdgpu_ps i16 @s_lshr_i16(i16 inreg %value, i16 inreg %amount) {
;
; GFX8-LABEL: s_lshr_i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshr_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_lshr_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
; GFX9-NEXT: s_lshr_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_lshr_i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX10PLUS-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, s1
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr i16 %value, %amount
@@ -688,19 +688,19 @@ define amdgpu_ps i16 @s_lshr_i16_15(i16 inreg %value) {
;
; GFX8-LABEL: s_lshr_i16_15:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshr_b32 s0, s0, 15
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_lshr_i16_15:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
; GFX9-NEXT: s_lshr_b32 s0, s0, 15
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_lshr_i16_15:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX10PLUS-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, 15
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr i16 %value, 15
@@ -852,15 +852,22 @@ define amdgpu_ps i32 @s_lshr_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amou
;
; GFX9-LABEL: s_lshr_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshr_b32 s1, s2, s1
+; GFX9-NEXT: s_lshr_b32 s0, s0, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_lshr_v2i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshrrev_b16 v0, s1, s0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s2, s1
+; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, s3
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr <2 x i16> %value, %amount
%cast = bitcast <2 x i16> %result to i32
@@ -1036,20 +1043,34 @@ define amdgpu_ps <2 x i32> @s_lshr_v4i16(<4 x i16> inreg %value, <4 x i16> inreg
;
; GFX9-LABEL: s_lshr_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, s3, v0
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, s2, v1
-; GFX9-NEXT: v_readfirstlane_b32 s0, v1
-; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-NEXT: s_lshr_b32 s2, s4, s2
+; GFX9-NEXT: s_lshr_b32 s0, s0, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshr_b32 s2, s2, s3
+; GFX9-NEXT: s_lshr_b32 s1, s1, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_lshr_v4i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshrrev_b16 v0, s2, s0
-; GFX10PLUS-NEXT: v_pk_lshrrev_b16 v1, s3, s1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10PLUS-NEXT: s_and_b32 s4, s0, 0xffff
+; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s2, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s4, s2
+; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, s5
+; GFX10PLUS-NEXT: s_and_b32 s4, s1, 0xffff
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s3, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s4, s3
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s1, s5
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s3, s1
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr <4 x i16> %value, %amount
%cast = bitcast <4 x i16> %result to <2 x i32>
@@ -1219,30 +1240,58 @@ define amdgpu_ps <4 x i32> @s_lshr_v8i16(<8 x i16> inreg %value, <8 x i16> inreg
;
; GFX9-LABEL: s_lshr_v8i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-NEXT: v_pk_lshrrev_b16 v0, s7, v0
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, s6, v1
-; GFX9-NEXT: v_pk_lshrrev_b16 v2, s5, v2
-; GFX9-NEXT: v_pk_lshrrev_b16 v3, s4, v3
-; GFX9-NEXT: v_readfirstlane_b32 s0, v3
-; GFX9-NEXT: v_readfirstlane_b32 s1, v2
-; GFX9-NEXT: v_readfirstlane_b32 s2, v1
-; GFX9-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s9, s4, 16
+; GFX9-NEXT: s_lshr_b32 s4, s8, s4
+; GFX9-NEXT: s_lshr_b32 s0, s0, s9
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX9-NEXT: s_and_b32 s4, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s8, s5, 16
+; GFX9-NEXT: s_lshr_b32 s4, s4, s5
+; GFX9-NEXT: s_lshr_b32 s1, s1, s8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s4, s1
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s5, s6, 16
+; GFX9-NEXT: s_lshr_b32 s4, s4, s6
+; GFX9-NEXT: s_lshr_b32 s2, s2, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT: s_and_b32 s4, s3, 0xffff
+; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: s_lshr_b32 s5, s7, 16
+; GFX9-NEXT: s_lshr_b32 s4, s4, s7
+; GFX9-NEXT: s_lshr_b32 s3, s3, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s4, s3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_lshr_v8i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshrrev_b16 v0, s4, s0
-; GFX10PLUS-NEXT: v_pk_lshrrev_b16 v1, s5, s1
-; GFX10PLUS-NEXT: v_pk_lshrrev_b16 v2, s6, s2
-; GFX10PLUS-NEXT: v_pk_lshrrev_b16 v3, s7, s3
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v2
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10PLUS-NEXT: s_and_b32 s8, s0, 0xffff
+; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s9, s4, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s8, s4
+; GFX10PLUS-NEXT: s_lshr_b32 s0, s0, s9
+; GFX10PLUS-NEXT: s_and_b32 s8, s1, 0xffff
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s9, s5, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s8, s5
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s1, s9
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s5, s1
+; GFX10PLUS-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s2, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s6, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s4, s6
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s2, s5
+; GFX10PLUS-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s3, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s6, s7, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s5, s7
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s3, s6
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s3, s5, s3
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr <8 x i16> %value, %amount
%cast = bitcast <8 x i16> %result to <4 x i32>
@@ -1575,110 +1624,115 @@ define i65 @v_lshr_i65(i65 %value, i65 %amount) {
; GFX6-LABEL: v_lshr_i65:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 1, v2
+; GFX6-NEXT: v_mov_b32_e32 v4, 1
; GFX6-NEXT: v_mov_b32_e32 v5, 0
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 64, v3
+; GFX6-NEXT: v_and_b32_e32 v4, 1, v2
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 64, v3
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xffffffc0, v3
; GFX6-NEXT: v_lshr_b64 v[6:7], v[0:1], v3
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], v2
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX6-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX6-NEXT: v_subrev_i32_e32 v6, vcc, 64, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v7, v9
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[4:5], v6
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], v8
+; GFX6-NEXT: v_lshr_b64 v[10:11], v[4:5], v3
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[4:5], v2
+; GFX6-NEXT: v_or_b32_e32 v6, v6, v8
+; GFX6-NEXT: v_or_b32_e32 v7, v7, v9
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v0, v6, v0, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v7, v2, vcc
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[4:5], v3
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v6, v1, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_lshr_i65:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v4, 1, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, 1
; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 64, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 1, v2
+; GFX8-NEXT: v_sub_u32_e32 v8, vcc, 64, v3
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0xffffffc0, v3
; GFX8-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], v2, v[4:5]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, 64, v3
-; GFX8-NEXT: v_or_b32_e32 v2, v7, v9
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], v6, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], v8, v[4:5]
+; GFX8-NEXT: v_lshrrev_b64 v[10:11], v3, v[4:5]
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v2, v[4:5]
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v8
+; GFX8-NEXT: v_or_b32_e32 v7, v7, v9
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v0, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v2, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v3, v[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v6, v1, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_lshr_i65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v4, 1, v2
+; GFX9-NEXT: v_mov_b32_e32 v4, 1
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_sub_u32_e32 v2, 64, v3
+; GFX9-NEXT: v_and_b32_e32 v4, 1, v2
+; GFX9-NEXT: v_sub_u32_e32 v8, 64, v3
+; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffc0, v3
; GFX9-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
-; GFX9-NEXT: v_lshlrev_b64 v[8:9], v2, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], v8, v[4:5]
+; GFX9-NEXT: v_lshrrev_b64 v[10:11], v3, v[4:5]
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], v2, v[4:5]
+; GFX9-NEXT: v_or_b32_e32 v6, v6, v8
+; GFX9-NEXT: v_or_b32_e32 v7, v7, v9
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX9-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX9-NEXT: v_subrev_u32_e32 v6, 64, v3
-; GFX9-NEXT: v_or_b32_e32 v2, v7, v9
-; GFX9-NEXT: v_lshrrev_b64 v[6:7], v6, v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v6, v0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v2, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], v3, v[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v6, v1, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_lshr_i65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v4, 1
; GFX10-NEXT: v_mov_b32_e32 v5, 0
; GFX10-NEXT: v_and_b32_e32 v4, 1, v2
; GFX10-NEXT: v_sub_nc_u32_e32 v2, 64, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 0xffffffc0, v3
; GFX10-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
-; GFX10-NEXT: v_subrev_nc_u32_e32 v10, 64, v3
; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v3
; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v3
; GFX10-NEXT: v_lshlrev_b64 v[8:9], v2, v[4:5]
; GFX10-NEXT: v_lshrrev_b64 v[10:11], v10, v[4:5]
; GFX10-NEXT: v_lshrrev_b64 v[4:5], v3, v[4:5]
-; GFX10-NEXT: v_or_b32_e32 v2, v7, v9
-; GFX10-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v10, v6, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v2, v1, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v0, s4
+; GFX10-NEXT: v_or_b32_e32 v2, v6, v8
+; GFX10-NEXT: v_or_b32_e32 v6, v7, v9
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v11, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v0, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v5, v1, s4
; GFX10-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_lshr_i65:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v4, 1, v2
+; GFX11-NEXT: v_dual_mov_b32 v4, 1 :: v_dual_mov_b32 v5, 0
+; GFX11-NEXT: v_and_b32_e32 v4, 1, v2
; GFX11-NEXT: v_sub_nc_u32_e32 v2, 64, v3
; GFX11-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
-; GFX11-NEXT: v_subrev_nc_u32_e32 v10, 64, v3
; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v3
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
; GFX11-NEXT: v_lshlrev_b64 v[8:9], v2, v[4:5]
+; GFX11-NEXT: v_or_b32_e32 v2, v6, v8
+; GFX11-NEXT: v_or_b32_e32 v6, v7, v9
+; GFX11-NEXT: v_add_nc_u32_e32 v10, 0xffffffc0, v3
; GFX11-NEXT: v_lshrrev_b64 v[10:11], v10, v[4:5]
; GFX11-NEXT: v_lshrrev_b64 v[4:5], v3, v[4:5]
-; GFX11-NEXT: v_or_b32_e32 v2, v7, v9
-; GFX11-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v10, v6, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v2, v1, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v11, v6, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v0, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0
; GFX11-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v5, v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = lshr i65 %value, %amount
ret i65 %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll
index 057c5fa35c2bd..b7e70a9d77915 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mad.ll
@@ -1,34 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=hawaii -o - %s | FileCheck --check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a -o - %s | FileCheck --check-prefix=GFX90A %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii -o - %s | FileCheck --check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a -o - %s | FileCheck --check-prefix=GFX90A %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -o - %s | FileCheck --check-prefix=GFX11 %s
define amdgpu_ps void @mad_i64_uniform(i64 inreg %a, i64 inreg %b, ptr addrspace(1) %out) {
; GFX7-LABEL: mad_i64_uniform:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mad_u64_u32 v[2:3], s[8:9], s0, v2, 0
+; GFX7-NEXT: v_mul_hi_u32 v2, s0, v2
+; GFX7-NEXT: s_mul_i32 s4, s0, s2
; GFX7-NEXT: s_mul_i32 s0, s0, s3
-; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mul_i32 s1, s1, s2
-; GFX7-NEXT: v_add_i32_e32 v3, vcc, s0, v3
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: v_add_i32_e32 v3, vcc, s1, v3
-; GFX7-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_readfirstlane_b32 s5, v2
+; GFX7-NEXT: s_add_u32 s0, s0, s5
+; GFX7-NEXT: s_add_u32 s5, s1, s0
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_endpgm
;
; GFX90A-LABEL: mad_i64_uniform:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_mul_i32 s3, s0, s3
-; GFX90A-NEXT: s_mul_hi_u32 s4, s0, s2
-; GFX90A-NEXT: s_add_i32 s3, s4, s3
+; GFX90A-NEXT: s_mul_i32 s4, s0, s2
+; GFX90A-NEXT: s_mul_hi_u32 s5, s0, s2
+; GFX90A-NEXT: s_mul_i32 s0, s0, s3
; GFX90A-NEXT: s_mul_i32 s1, s1, s2
-; GFX90A-NEXT: s_add_i32 s3, s3, s1
-; GFX90A-NEXT: s_mul_i32 s0, s0, s2
-; GFX90A-NEXT: v_mov_b32_e32 v2, s0
-; GFX90A-NEXT: v_mov_b32_e32 v3, s3
+; GFX90A-NEXT: s_add_u32 s0, s1, s0
+; GFX90A-NEXT: s_add_i32 s5, s5, s0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX90A-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
index 8781e8f1a9aec..b882e0c217921 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
@@ -117,9 +117,13 @@ define i64 @test_abs_i64(i64 %a) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_sub_nc_u64_e32 v[2:3], 0, v[0:1]
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_max_i64 v[0:1], v[0:1], v[2:3]
+; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
+; CHECK-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call i64 @llvm.abs.i64(i64 %a, i1 0)
ret i64 %r
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
index 06fa6ba8d6113..2775beb4c38b1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GFX12 %s
; Test end to end matching of addressing modes when MUBUF is used for
; global memory.
@@ -315,11 +315,11 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_sgpr_offset(ptr addrspace(1) inreg %
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s5, s4, 31
; GFX6-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
@@ -329,11 +329,11 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_sgpr_offset(ptr addrspace(1) inreg %
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s5, s4, 31
; GFX7-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
@@ -345,7 +345,8 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_sgpr_offset(ptr addrspace(1) inreg %
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: s_lshl_b64 s[0:1], s[4:5], 2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
+; GFX12-NEXT: s_add_co_u32 s0, s2, s0
+; GFX12-NEXT: s_add_co_ci_u32 s1, s3, s1
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(1) %ptr, i32 %soffset
@@ -357,31 +358,33 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset(ptr addrspace(1) %ptr, i
; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s3, s2, 31
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_lshl_b64 s[4:5], s[2:3], 2
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
+; GFX6-NEXT: s_mov_b32 s2, 0
; GFX6-NEXT: v_mov_b32_e32 v2, 0
-; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s3, s2, 31
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_lshl_b64 s[4:5], s[2:3], 2
+; GFX7-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
+; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
-; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_endpgm
;
; GFX12-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(1) %ptr, i32 %soffset
@@ -394,9 +397,9 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s3, s2, 31
; GFX6-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, 0
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
; GFX6-NEXT: s_endpgm
;
@@ -404,20 +407,22 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s3, s2, 31
; GFX7-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
; GFX7-NEXT: s_endpgm
;
; GFX12-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:1024
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 %soffset
@@ -431,9 +436,9 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s3, s2, 31
; GFX6-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, 0
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
; GFX6-NEXT: s_endpgm
;
@@ -441,20 +446,22 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s3, s2, 31
; GFX7-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
; GFX7-NEXT: s_endpgm
;
; GFX12-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:1024
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 256
@@ -468,11 +475,11 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
; GFX6: ; %bb.0:
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT: s_endpgm
;
@@ -480,23 +487,24 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
; GFX7: ; %bb.0:
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT: s_endpgm
;
; GFX12-LABEL: mubuf_store_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(1) %ptr, i32 %voffset
@@ -509,12 +517,12 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
; GFX6: ; %bb.0:
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
-; GFX6-NEXT: s_movk_i32 s4, 0x3ffc
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_movk_i32 s4, 0x3ffc
; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX6-NEXT: s_endpgm
;
@@ -522,24 +530,25 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
; GFX7: ; %bb.0:
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_movk_i32 s4, 0x3ffc
; GFX7-NEXT: v_mov_b32_e32 v2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_movk_i32 s4, 0x3ffc
; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX7-NEXT: s_endpgm
;
; GFX12-LABEL: mubuf_store_sgpr_ptr_vgpr_offset_offset4095:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:16380
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 %voffset
@@ -552,12 +561,12 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
; GFX6: ; %bb.0:
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
-; GFX6-NEXT: s_movk_i32 s4, 0x3ffc
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_movk_i32 s4, 0x3ffc
; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX6-NEXT: s_endpgm
;
@@ -565,24 +574,25 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
; GFX7: ; %bb.0:
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_movk_i32 s4, 0x3ffc
; GFX7-NEXT: v_mov_b32_e32 v2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_movk_i32 s4, 0x3ffc
; GFX7-NEXT: buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX7-NEXT: s_endpgm
;
; GFX12-LABEL: mubuf_store_sgpr_ptr_offset4095_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:16380
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 4095
@@ -903,10 +913,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_sgpr_offset(ptr addrspace(1) inreg %
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s5, s4, 31
; GFX6-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
@@ -917,10 +927,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_sgpr_offset(ptr addrspace(1) inreg %
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s5, s4, 31
; GFX7-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
@@ -933,7 +943,8 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_sgpr_offset(ptr addrspace(1) inreg %
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: s_lshl_b64 s[0:1], s[4:5], 2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
+; GFX12-NEXT: s_add_co_u32 s0, s2, s0
+; GFX12-NEXT: s_add_co_ci_u32 s1, s3, s1
; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -946,20 +957,20 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset(ptr addrspace(1) %ptr, i
; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s3, s2, 31
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_lshl_b64 s[4:5], s[2:3], 2
-; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
+; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s3, s2, 31
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_lshl_b64 s[4:5], s[2:3], 2
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: ; return to shader part epilog
;
@@ -968,9 +979,10 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset(ptr addrspace(1) %ptr, i
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -984,8 +996,8 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s3, s2, 31
; GFX6-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: ; return to shader part epilog
@@ -994,8 +1006,8 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s3, s2, 31
; GFX7-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: ; return to shader part epilog
@@ -1005,9 +1017,10 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:1024 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1022,8 +1035,8 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
; GFX6: ; %bb.0:
; GFX6-NEXT: s_ashr_i32 s3, s2, 31
; GFX6-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: ; return to shader part epilog
@@ -1032,8 +1045,8 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
; GFX7: ; %bb.0:
; GFX7-NEXT: s_ashr_i32 s3, s2, 31
; GFX7-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: ; return to shader part epilog
@@ -1043,9 +1056,10 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:1024 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1060,10 +1074,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
; GFX6: ; %bb.0:
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: ; return to shader part epilog
@@ -1072,10 +1086,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
; GFX7: ; %bb.0:
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: ; return to shader part epilog
@@ -1083,11 +1097,12 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
; GFX12-LABEL: mubuf_load_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1101,10 +1116,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
; GFX6: ; %bb.0:
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_movk_i32 s4, 0x3ffc
; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -1114,10 +1129,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
; GFX7: ; %bb.0:
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_movk_i32 s4, 0x3ffc
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -1126,11 +1141,12 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
; GFX12-LABEL: mubuf_load_sgpr_ptr_vgpr_offset_offset4095:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:16380 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1144,10 +1160,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
; GFX6: ; %bb.0:
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_movk_i32 s4, 0x3ffc
; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -1157,10 +1173,10 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
; GFX7: ; %bb.0:
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_movk_i32 s4, 0x3ffc
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -1169,11 +1185,12 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
; GFX12-LABEL: mubuf_load_sgpr_ptr_offset4095_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:16380 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1361,11 +1378,11 @@ define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_vgpr_offset(ptr addrspace(1) in
; GFX6: ; %bb.0:
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[1:2], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: v_mov_b32_e32 v0, 2
+; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_atomic_add v0, v[1:2], s[0:3], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
@@ -1376,11 +1393,11 @@ define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_vgpr_offset(ptr addrspace(1) in
; GFX7: ; %bb.0:
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[1:2], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: v_mov_b32_e32 v0, 2
+; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_atomic_add v0, v[1:2], s[0:3], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
@@ -1389,12 +1406,13 @@ define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_vgpr_offset(ptr addrspace(1) in
; GFX12-LABEL: mubuf_atomicrmw_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_mov_b32_e32 v2, 2
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX12-NEXT: v_mov_b32_e32 v2, 2
; GFX12-NEXT: global_atomic_add_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
@@ -1591,10 +1609,10 @@ define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_vgpr_offset(ptr addrspace(1) inre
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s0, s2
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s1, s3
; GFX6-NEXT: s_mov_b32 s2, 0
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
@@ -1607,10 +1625,10 @@ define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_vgpr_offset(ptr addrspace(1) inre
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s1, s3
; GFX7-NEXT: s_mov_b32 s2, 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
@@ -1619,13 +1637,14 @@ define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_vgpr_offset(ptr addrspace(1) inre
;
; GFX12-LABEL: mubuf_cmpxchg_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, s2
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_mov_b32_e32 v5, s3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v4, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v5, v1, vcc_lo
; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index fe142ade17951..6f7cf8d8657c8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -1,13 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1310 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX13 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1310 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX13 %s
define amdgpu_ps i16 @s_mul_i16(i16 inreg %num, i16 inreg %den) {
; GCN-LABEL: s_mul_i16:
@@ -109,23 +109,35 @@ define i16 @v_mul_i16(i16 %num, i16 %den) {
}
define amdgpu_ps zeroext i16 @s_mul_i16_zeroext(i16 inreg zeroext %num, i16 inreg zeroext %den) {
-; GCN-LABEL: s_mul_i16_zeroext:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_mul_i32 s0, s0, s1
-; GCN-NEXT: s_and_b32 s0, s0, 0xffff
-; GCN-NEXT: ; return to shader part epilog
+; GFX7-LABEL: s_mul_i16_zeroext:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_mul_i32 s0, s0, s1
+; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX7-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_mul_i16_zeroext:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_mul_i32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX9-LABEL: s_mul_i16_zeroext:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_mul_i16_zeroext:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_mul_i32 s0, s0, s1
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10PLUS-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_mul_i16_zeroext:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_mul_i32 s0, s0, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX12-NEXT: s_and_b32 s0, 0xffff, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: s_mul_i16_zeroext:
@@ -133,14 +145,14 @@ define amdgpu_ps zeroext i16 @s_mul_i16_zeroext(i16 inreg zeroext %num, i16 inre
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s0, s0, s1
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX1250-NEXT: s_and_b32 s0, 0xffff, s0
; GFX1250-NEXT: ; return to shader part epilog
;
; GFX13-LABEL: s_mul_i16_zeroext:
; GFX13: ; %bb.0:
; GFX13-NEXT: s_mul_i32 s0, s0, s1
; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX13-NEXT: s_and_b32 s0, 0xffff, s0
; GFX13-NEXT: ; return to shader part epilog
%result = mul i16 %num, %den
ret i16 %result
@@ -445,42 +457,59 @@ define amdgpu_ps <2 x i16> @s_mul_v2i16(<2 x i16> inreg %num, <2 x i16> inreg %d
; GFX9-LABEL: s_mul_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, s0, v0
-; GFX9-NEXT: v_pk_mad_u16 v0, s0, v0, v1
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, s0, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_add_i32 s0, s0, s0
+; GFX9-NEXT: s_add_i32 s1, s1, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_mul_v2i16:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: v_pk_mul_lo_u16 v0, s0, s1
-; GFX10PLUS-NEXT: v_pk_mad_u16 v0, s0, s1, v0
; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10PLUS-NEXT: s_add_i32 s0, s0, s0
+; GFX10PLUS-NEXT: s_add_i32 s1, s1, s1
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_mul_v2i16:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_pk_mul_lo_u16 v0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_mad_u16 v0, s0, s1, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_lshr_b32 s1, s0, 16
+; GFX12-NEXT: s_add_co_i32 s0, s0, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s1
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: s_mul_v2i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_mul_lo_u16 v0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mad_u16 v0, s0, s1, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_lshr_b32 s1, s0, 16
+; GFX1250-NEXT: s_add_co_i32 s0, s0, s0
+; GFX1250-NEXT: s_add_co_i32 s1, s1, s1
+; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
;
; GFX13-LABEL: s_mul_v2i16:
; GFX13: ; %bb.0:
; GFX13-NEXT: v_pk_mul_lo_u16 v0, s0, s1
-; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_pk_mad_u16 v0, s0, s1, v0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX13-NEXT: v_readfirstlane_b32 s0, v0
+; GFX13-NEXT: s_lshr_b32 s1, s0, 16
+; GFX13-NEXT: s_add_co_i32 s0, s0, s0
+; GFX13-NEXT: s_add_co_i32 s1, s1, s1
+; GFX13-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX13-NEXT: ; return to shader part epilog
%mul = mul <2 x i16> %num, %den
%result = add <2 x i16> %mul, %mul
@@ -639,35 +668,38 @@ define amdgpu_cs i33 @s_mul_i33(i33 inreg %num, i33 inreg %den) {
; GFX7-LABEL: s_mul_i33:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s0, v0, 0
+; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX7-NEXT: s_mul_i32 s4, s0, s2
; GFX7-NEXT: s_mul_i32 s0, s0, s3
; GFX7-NEXT: s_mul_i32 s1, s1, s2
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s0, v1
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s1, v1
-; GFX7-NEXT: v_readfirstlane_b32 s1, v1
-; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: v_readfirstlane_b32 s5, v0
+; GFX7-NEXT: s_add_u32 s0, s0, s5
+; GFX7-NEXT: s_add_u32 s1, s1, s0
+; GFX7-NEXT: s_mov_b32 s0, s4
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_mul_i33:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s0, v0, 0
+; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX8-NEXT: s_mul_i32 s4, s0, s2
; GFX8-NEXT: s_mul_i32 s0, s0, s3
; GFX8-NEXT: s_mul_i32 s1, s1, s2
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s5, v0
+; GFX8-NEXT: s_add_u32 s0, s0, s5
+; GFX8-NEXT: s_add_u32 s1, s1, s0
+; GFX8-NEXT: s_mov_b32 s0, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_mul_i33:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mul_i32 s3, s0, s3
-; GFX9-NEXT: s_mul_hi_u32 s4, s0, s2
-; GFX9-NEXT: s_add_i32 s3, s4, s3
+; GFX9-NEXT: s_mul_i32 s4, s0, s2
+; GFX9-NEXT: s_mul_hi_u32 s5, s0, s2
+; GFX9-NEXT: s_mul_i32 s0, s0, s3
+; GFX9-NEXT: s_add_u32 s0, s0, s5
; GFX9-NEXT: s_mul_i32 s1, s1, s2
-; GFX9-NEXT: s_add_i32 s1, s3, s1
-; GFX9-NEXT: s_mul_i32 s0, s0, s2
+; GFX9-NEXT: s_add_u32 s1, s1, s0
+; GFX9-NEXT: s_mov_b32 s0, s4
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_mul_i33:
@@ -703,35 +735,38 @@ define amdgpu_ps i64 @s_mul_i64(i64 inreg %num, i64 inreg %den) {
; GFX7-LABEL: s_mul_i64:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s0, v0, 0
+; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX7-NEXT: s_mul_i32 s4, s0, s2
; GFX7-NEXT: s_mul_i32 s0, s0, s3
; GFX7-NEXT: s_mul_i32 s1, s1, s2
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s0, v1
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s1, v1
-; GFX7-NEXT: v_readfirstlane_b32 s1, v1
-; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: v_readfirstlane_b32 s5, v0
+; GFX7-NEXT: s_add_u32 s0, s0, s5
+; GFX7-NEXT: s_add_u32 s1, s1, s0
+; GFX7-NEXT: s_mov_b32 s0, s4
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_mul_i64:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s0, v0, 0
+; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX8-NEXT: s_mul_i32 s4, s0, s2
; GFX8-NEXT: s_mul_i32 s0, s0, s3
; GFX8-NEXT: s_mul_i32 s1, s1, s2
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s5, v0
+; GFX8-NEXT: s_add_u32 s0, s0, s5
+; GFX8-NEXT: s_add_u32 s1, s1, s0
+; GFX8-NEXT: s_mov_b32 s0, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_mul_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mul_i32 s3, s0, s3
-; GFX9-NEXT: s_mul_hi_u32 s4, s0, s2
-; GFX9-NEXT: s_add_i32 s3, s4, s3
+; GFX9-NEXT: s_mul_i32 s4, s0, s2
+; GFX9-NEXT: s_mul_hi_u32 s5, s0, s2
+; GFX9-NEXT: s_mul_i32 s0, s0, s3
+; GFX9-NEXT: s_add_u32 s0, s0, s5
; GFX9-NEXT: s_mul_i32 s1, s1, s2
-; GFX9-NEXT: s_add_i32 s1, s3, s1
-; GFX9-NEXT: s_mul_i32 s0, s0, s2
+; GFX9-NEXT: s_add_u32 s1, s1, s0
+; GFX9-NEXT: s_mov_b32 s0, s4
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_mul_i64:
@@ -802,11 +837,12 @@ define i64 @v_mul_i64(i64 %num, i64 %den) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u32 v4, v1, v2
+; GFX12-NEXT: v_mul_hi_u32 v4, v0, v2
; GFX12-NEXT: v_mul_lo_u32 v3, v0, v3
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v1, v1, v3, v4
+; GFX12-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX12-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_add3_u32 v1, v3, v1, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_mul_i64:
@@ -823,11 +859,12 @@ define i64 @v_mul_i64(i64 %num, i64 %den) {
; GFX13-NEXT: s_wait_samplecnt 0x0
; GFX13-NEXT: s_wait_bvhcnt 0x0
; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_mul_lo_u32 v4, v1, v2
+; GFX13-NEXT: v_mul_hi_u32 v4, v0, v2
; GFX13-NEXT: v_mul_lo_u32 v3, v0, v3
-; GFX13-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
-; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-NEXT: v_add3_u32 v1, v1, v3, v4
+; GFX13-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX13-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_add3_u32 v1, v3, v1, v4
; GFX13-NEXT: s_set_pc_i64 s[30:31]
%result = mul i64 %num, %den
ret i64 %result
@@ -836,192 +873,155 @@ define i64 @v_mul_i64(i64 %num, i64 %den) {
define amdgpu_ps <3 x i32> @s_mul_i96(i96 inreg %num, i96 inreg %den) {
; GFX7-LABEL: s_mul_i96:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s0, v3, 0
-; GFX7-NEXT: v_mov_b32_e32 v2, 0
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: s_mul_i32 s4, s4, s2
-; GFX7-NEXT: v_mad_u64_u32 v[3:4], s[6:7], s1, v3, v[1:2]
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
-; GFX7-NEXT: v_mad_u64_u32 v[1:2], s[6:7], s0, v6, v[1:2]
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[6:7], s3, v4, 0
-; GFX7-NEXT: v_addc_u32_e64 v3, s[6:7], 0, 0, vcc
-; GFX7-NEXT: v_mad_u64_u32 v[2:3], s[6:7], s1, v6, v[2:3]
-; GFX7-NEXT: v_add_i32_e32 v5, vcc, s4, v5
-; GFX7-NEXT: v_mov_b32_e32 v3, s0
-; GFX7-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s5, v3, v[4:5]
-; GFX7-NEXT: v_readfirstlane_b32 s0, v0
-; GFX7-NEXT: v_readfirstlane_b32 s1, v1
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; GFX7-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, s3
+; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, s4
+; GFX7-NEXT: v_mul_hi_u32 v1, s0, v1
+; GFX7-NEXT: s_mul_i32 s5, s0, s5
+; GFX7-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7-NEXT: s_mul_i32 s8, s1, s4
+; GFX7-NEXT: v_mov_b32_e32 v0, s1
+; GFX7-NEXT: s_add_u32 s5, s8, s5
+; GFX7-NEXT: s_mul_i32 s2, s2, s3
+; GFX7-NEXT: v_mul_hi_u32 v0, v0, s3
+; GFX7-NEXT: s_mul_i32 s6, s0, s3
+; GFX7-NEXT: s_add_u32 s2, s2, s5
+; GFX7-NEXT: s_mul_i32 s0, s0, s4
+; GFX7-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7-NEXT: s_add_u32 s0, s0, s7
+; GFX7-NEXT: s_addc_u32 s2, s4, s2
+; GFX7-NEXT: s_mul_i32 s1, s1, s3
+; GFX7-NEXT: v_readfirstlane_b32 s3, v0
+; GFX7-NEXT: s_add_u32 s1, s1, s0
+; GFX7-NEXT: s_addc_u32 s2, s3, s2
+; GFX7-NEXT: s_mov_b32 s0, s6
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_mul_i96:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s0, v3, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: s_mul_i32 s4, s4, s2
-; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[6:7], s1, v3, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
-; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[6:7], s0, v6, v[1:2]
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[6:7], s3, v4, 0
-; GFX8-NEXT: v_addc_u32_e64 v3, s[6:7], 0, 0, vcc
-; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[6:7], s1, v6, v[2:3]
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT: v_mov_b32_e32 v3, s0
-; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s5, v3, v[4:5]
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT: v_readfirstlane_b32 s2, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s4
+; GFX8-NEXT: v_mul_hi_u32 v1, s0, v1
+; GFX8-NEXT: s_mul_i32 s5, s0, s5
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
+; GFX8-NEXT: s_mul_i32 s8, s1, s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: s_add_u32 s5, s8, s5
+; GFX8-NEXT: s_mul_i32 s2, s2, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s3
+; GFX8-NEXT: s_mul_i32 s6, s0, s3
+; GFX8-NEXT: s_add_u32 s2, s2, s5
+; GFX8-NEXT: s_mul_i32 s0, s0, s4
+; GFX8-NEXT: v_readfirstlane_b32 s4, v1
+; GFX8-NEXT: s_add_u32 s0, s0, s7
+; GFX8-NEXT: s_addc_u32 s2, s4, s2
+; GFX8-NEXT: s_mul_i32 s1, s1, s3
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_add_u32 s1, s1, s0
+; GFX8-NEXT: s_addc_u32 s2, s3, s2
+; GFX8-NEXT: s_mov_b32 s0, s6
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_mul_i96:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mul_i32 s10, s1, s3
-; GFX9-NEXT: s_mul_hi_u32 s11, s0, s3
-; GFX9-NEXT: s_mul_hi_u32 s9, s1, s3
-; GFX9-NEXT: s_add_u32 s10, s10, s11
-; GFX9-NEXT: s_mul_i32 s7, s0, s4
-; GFX9-NEXT: s_addc_u32 s9, s9, 0
-; GFX9-NEXT: s_mul_hi_u32 s8, s0, s4
-; GFX9-NEXT: s_add_u32 s7, s7, s10
-; GFX9-NEXT: s_addc_u32 s8, s8, 0
-; GFX9-NEXT: s_add_i32 s9, s9, s8
-; GFX9-NEXT: s_mul_i32 s1, s1, s4
-; GFX9-NEXT: s_mul_i32 s2, s3, s2
-; GFX9-NEXT: s_mul_i32 s4, s5, s0
-; GFX9-NEXT: s_mov_b32 s6, 0
-; GFX9-NEXT: s_add_i32 s1, s1, s9
-; GFX9-NEXT: s_add_i32 s4, s4, s2
-; GFX9-NEXT: s_add_i32 s2, s1, s4
-; GFX9-NEXT: s_mul_i32 s0, s0, s3
-; GFX9-NEXT: s_mov_b32 s1, s6
-; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX9-NEXT: s_mul_i32 s5, s0, s5
+; GFX9-NEXT: s_mul_i32 s8, s1, s4
+; GFX9-NEXT: s_add_u32 s5, s8, s5
+; GFX9-NEXT: s_mul_i32 s2, s2, s3
+; GFX9-NEXT: s_mul_hi_u32 s7, s0, s3
+; GFX9-NEXT: s_add_u32 s2, s2, s5
+; GFX9-NEXT: s_mul_i32 s5, s0, s4
+; GFX9-NEXT: s_mul_i32 s6, s0, s3
+; GFX9-NEXT: s_mul_hi_u32 s0, s0, s4
+; GFX9-NEXT: s_add_u32 s4, s5, s7
+; GFX9-NEXT: s_addc_u32 s0, s0, s2
+; GFX9-NEXT: s_mul_i32 s2, s1, s3
+; GFX9-NEXT: s_mul_hi_u32 s3, s1, s3
+; GFX9-NEXT: s_add_u32 s1, s2, s4
+; GFX9-NEXT: s_addc_u32 s2, s3, s0
+; GFX9-NEXT: s_mov_b32 s0, s6
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_mul_i96:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_mul_i32 s8, s1, s3
-; GFX10PLUS-NEXT: s_mul_hi_u32 s9, s0, s3
-; GFX10PLUS-NEXT: s_mul_hi_u32 s10, s1, s3
-; GFX10PLUS-NEXT: s_mul_i32 s7, s0, s4
-; GFX10PLUS-NEXT: s_add_u32 s8, s8, s9
-; GFX10PLUS-NEXT: s_mul_hi_u32 s6, s0, s4
-; GFX10PLUS-NEXT: s_addc_u32 s9, s10, 0
-; GFX10PLUS-NEXT: s_add_u32 s7, s7, s8
-; GFX10PLUS-NEXT: s_addc_u32 s6, s6, 0
-; GFX10PLUS-NEXT: s_mul_i32 s1, s1, s4
-; GFX10PLUS-NEXT: s_add_i32 s9, s9, s6
-; GFX10PLUS-NEXT: s_mov_b32 s6, 0
-; GFX10PLUS-NEXT: s_mul_i32 s2, s3, s2
-; GFX10PLUS-NEXT: s_mul_i32 s5, s5, s0
-; GFX10PLUS-NEXT: s_add_i32 s4, s1, s9
-; GFX10PLUS-NEXT: s_mul_i32 s0, s0, s3
-; GFX10PLUS-NEXT: s_mov_b32 s1, s6
-; GFX10PLUS-NEXT: s_add_i32 s5, s5, s2
-; GFX10PLUS-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
-; GFX10PLUS-NEXT: s_add_i32 s2, s4, s5
+; GFX10PLUS-NEXT: s_mul_i32 s6, s0, s5
+; GFX10PLUS-NEXT: s_mul_i32 s7, s1, s4
+; GFX10PLUS-NEXT: s_mul_i32 s2, s2, s3
+; GFX10PLUS-NEXT: s_add_i32 s6, s6, s7
+; GFX10PLUS-NEXT: s_mul_hi_u32 s7, s0, s3
+; GFX10PLUS-NEXT: s_add_i32 s6, s6, s2
+; GFX10PLUS-NEXT: s_mul_i32 s2, s0, s4
+; GFX10PLUS-NEXT: s_mul_i32 s5, s0, s3
+; GFX10PLUS-NEXT: s_mul_hi_u32 s0, s0, s4
+; GFX10PLUS-NEXT: s_add_u32 s2, s2, s7
+; GFX10PLUS-NEXT: s_mul_i32 s4, s1, s3
+; GFX10PLUS-NEXT: s_addc_u32 s0, s0, s6
+; GFX10PLUS-NEXT: s_mul_hi_u32 s3, s1, s3
+; GFX10PLUS-NEXT: s_add_u32 s1, s4, s2
+; GFX10PLUS-NEXT: s_addc_u32 s2, s3, s0
+; GFX10PLUS-NEXT: s_mov_b32 s0, s5
; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_mul_i96:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mov_b32 s9, 0
-; GFX12-NEXT: s_mov_b32 s8, s3
-; GFX12-NEXT: s_mov_b32 s10, s1
-; GFX12-NEXT: s_mov_b32 s1, s9
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mul_u64 s[12:13], s[0:1], s[8:9]
-; GFX12-NEXT: s_mul_u64 s[14:15], s[10:11], s[8:9]
-; GFX12-NEXT: s_mov_b32 s16, s13
-; GFX12-NEXT: s_mov_b32 s17, s9
-; GFX12-NEXT: s_mov_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, s9
-; GFX12-NEXT: s_add_nc_u64 s[14:15], s[14:15], s[16:17]
-; GFX12-NEXT: s_mul_u64 s[16:17], s[0:1], s[4:5]
-; GFX12-NEXT: s_mov_b32 s18, s14
-; GFX12-NEXT: s_mov_b32 s19, s9
-; GFX12-NEXT: s_mul_u64 s[4:5], s[10:11], s[4:5]
-; GFX12-NEXT: s_add_nc_u64 s[10:11], s[16:17], s[18:19]
-; GFX12-NEXT: s_mul_i32 s2, s8, s2
-; GFX12-NEXT: s_add_co_i32 s1, s15, s11
-; GFX12-NEXT: s_mov_b32 s13, s9
-; GFX12-NEXT: s_add_co_i32 s3, s4, s1
-; GFX12-NEXT: s_mul_i32 s4, s6, s0
-; GFX12-NEXT: s_mov_b32 s0, s9
-; GFX12-NEXT: s_mov_b32 s1, s10
-; GFX12-NEXT: s_add_co_i32 s4, s4, s2
-; GFX12-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
-; GFX12-NEXT: s_add_co_i32 s2, s3, s4
+; GFX12-NEXT: s_mul_i32 s6, s0, s5
+; GFX12-NEXT: s_mul_i32 s7, s1, s4
+; GFX12-NEXT: s_mul_i32 s2, s2, s3
+; GFX12-NEXT: s_add_co_i32 s6, s6, s7
+; GFX12-NEXT: s_mul_hi_u32 s7, s0, s3
+; GFX12-NEXT: s_add_co_i32 s6, s6, s2
+; GFX12-NEXT: s_mul_i32 s2, s0, s4
+; GFX12-NEXT: s_mul_i32 s5, s0, s3
+; GFX12-NEXT: s_mul_hi_u32 s0, s0, s4
+; GFX12-NEXT: s_add_co_u32 s2, s2, s7
+; GFX12-NEXT: s_mul_i32 s4, s1, s3
+; GFX12-NEXT: s_add_co_ci_u32 s0, s0, s6
+; GFX12-NEXT: s_mul_hi_u32 s3, s1, s3
+; GFX12-NEXT: s_add_co_u32 s1, s4, s2
+; GFX12-NEXT: s_add_co_ci_u32 s2, s3, s0
+; GFX12-NEXT: s_mov_b32 s0, s5
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: s_mul_i96:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[10:11], 0xffffffff
-; GFX1250-NEXT: s_mov_b32 s8, s3
-; GFX1250-NEXT: s_mov_b32 s9, 0
-; GFX1250-NEXT: s_and_b64 s[10:11], s[0:1], s[10:11]
-; GFX1250-NEXT: s_mov_b32 s12, s1
-; GFX1250-NEXT: s_mov_b32 s13, s9
-; GFX1250-NEXT: s_mul_u64 s[14:15], s[10:11], s[8:9]
-; GFX1250-NEXT: s_mul_u64 s[12:13], s[12:13], s[8:9]
-; GFX1250-NEXT: s_mov_b32 s16, s15
-; GFX1250-NEXT: s_mov_b32 s17, s9
-; GFX1250-NEXT: s_mov_b32 s6, s5
-; GFX1250-NEXT: s_add_nc_u64 s[12:13], s[12:13], s[16:17]
-; GFX1250-NEXT: s_mov_b32 s5, s9
-; GFX1250-NEXT: s_mov_b32 s16, s12
-; GFX1250-NEXT: s_mul_u64 s[10:11], s[10:11], s[4:5]
-; GFX1250-NEXT: s_mul_i32 s1, s1, s4
-; GFX1250-NEXT: s_add_nc_u64 s[10:11], s[10:11], s[16:17]
-; GFX1250-NEXT: s_mul_i32 s2, s8, s2
-; GFX1250-NEXT: s_add_co_i32 s3, s13, s11
-; GFX1250-NEXT: s_mul_i32 s4, s6, s0
-; GFX1250-NEXT: s_add_co_i32 s3, s1, s3
-; GFX1250-NEXT: s_mov_b32 s0, s9
-; GFX1250-NEXT: s_mov_b32 s1, s10
-; GFX1250-NEXT: s_mov_b32 s15, s9
-; GFX1250-NEXT: s_add_co_i32 s4, s4, s2
-; GFX1250-NEXT: s_or_b64 s[0:1], s[14:15], s[0:1]
-; GFX1250-NEXT: s_add_co_i32 s2, s3, s4
+; GFX1250-NEXT: s_mul_i32 s6, s0, s5
+; GFX1250-NEXT: s_mul_i32 s7, s1, s4
+; GFX1250-NEXT: s_mul_i32 s2, s2, s3
+; GFX1250-NEXT: s_add_co_i32 s6, s6, s7
+; GFX1250-NEXT: s_mul_hi_u32 s7, s0, s3
+; GFX1250-NEXT: s_add_co_i32 s6, s6, s2
+; GFX1250-NEXT: s_mul_i32 s2, s0, s4
+; GFX1250-NEXT: s_mul_i32 s5, s0, s3
+; GFX1250-NEXT: s_mul_hi_u32 s0, s0, s4
+; GFX1250-NEXT: s_add_co_u32 s2, s2, s7
+; GFX1250-NEXT: s_mul_i32 s4, s1, s3
+; GFX1250-NEXT: s_add_co_ci_u32 s0, s0, s6
+; GFX1250-NEXT: s_mul_hi_u32 s3, s1, s3
+; GFX1250-NEXT: s_add_co_u32 s1, s4, s2
+; GFX1250-NEXT: s_add_co_ci_u32 s2, s3, s0
+; GFX1250-NEXT: s_mov_b32 s0, s5
; GFX1250-NEXT: ; return to shader part epilog
;
; GFX13-LABEL: s_mul_i96:
; GFX13: ; %bb.0:
-; GFX13-NEXT: s_mov_b64 s[10:11], 0xffffffff
-; GFX13-NEXT: s_mov_b32 s8, s3
-; GFX13-NEXT: s_mov_b32 s9, 0
-; GFX13-NEXT: s_and_b64 s[10:11], s[0:1], s[10:11]
-; GFX13-NEXT: s_mov_b32 s12, s1
-; GFX13-NEXT: s_mul_u64 s[14:15], s[10:11], s[8:9]
-; GFX13-NEXT: s_mov_b32 s13, s9
-; GFX13-NEXT: s_mov_b32 s18, s15
-; GFX13-NEXT: s_mul_u64 s[16:17], s[12:13], s[8:9]
-; GFX13-NEXT: s_mov_b32 s19, s9
-; GFX13-NEXT: s_mov_b32 s6, s5
-; GFX13-NEXT: s_mov_b32 s5, s9
-; GFX13-NEXT: s_add_nc_u64 s[16:17], s[16:17], s[18:19]
-; GFX13-NEXT: s_mul_u64 s[10:11], s[10:11], s[4:5]
-; GFX13-NEXT: s_mov_b32 s18, s16
-; GFX13-NEXT: s_mul_u64 s[4:5], s[12:13], s[4:5]
-; GFX13-NEXT: s_add_nc_u64 s[10:11], s[10:11], s[18:19]
-; GFX13-NEXT: s_mul_i32 s2, s8, s2
-; GFX13-NEXT: s_add_co_i32 s1, s17, s11
-; GFX13-NEXT: s_mov_b32 s15, s9
-; GFX13-NEXT: s_add_co_i32 s3, s4, s1
-; GFX13-NEXT: s_mul_i32 s4, s6, s0
-; GFX13-NEXT: s_mov_b32 s0, s9
-; GFX13-NEXT: s_mov_b32 s1, s10
-; GFX13-NEXT: s_add_co_i32 s4, s4, s2
-; GFX13-NEXT: s_or_b64 s[0:1], s[14:15], s[0:1]
-; GFX13-NEXT: s_add_co_i32 s2, s3, s4
+; GFX13-NEXT: s_mul_i32 s6, s0, s5
+; GFX13-NEXT: s_mul_i32 s7, s1, s4
+; GFX13-NEXT: s_mul_i32 s2, s2, s3
+; GFX13-NEXT: s_add_co_i32 s6, s6, s7
+; GFX13-NEXT: s_mul_hi_u32 s7, s0, s3
+; GFX13-NEXT: s_add_co_i32 s6, s6, s2
+; GFX13-NEXT: s_mul_i32 s2, s0, s4
+; GFX13-NEXT: s_mul_i32 s5, s0, s3
+; GFX13-NEXT: s_mul_hi_u32 s0, s0, s4
+; GFX13-NEXT: s_add_co_u32 s2, s2, s7
+; GFX13-NEXT: s_mul_i32 s4, s1, s3
+; GFX13-NEXT: s_add_co_ci_u32 s0, s0, s6
+; GFX13-NEXT: s_mul_hi_u32 s3, s1, s3
+; GFX13-NEXT: s_add_co_u32 s1, s4, s2
+; GFX13-NEXT: s_add_co_ci_u32 s2, s3, s0
+; GFX13-NEXT: s_mov_b32 s0, s5
; GFX13-NEXT: ; return to shader part epilog
%result = mul i96 %num, %den
%cast = bitcast i96 %result to <3 x i32>
@@ -1136,241 +1136,280 @@ define i96 @v_mul_i96(i96 %num, i96 %den) {
define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
; GFX7-LABEL: s_mul_i128:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: v_mad_u64_u32 v[3:4], s[8:9], s4, v0, 0
-; GFX7-NEXT: s_mul_i32 s3, s4, s3
-; GFX7-NEXT: v_mov_b32_e32 v5, s4
-; GFX7-NEXT: s_mul_i32 s8, s5, s2
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, s3, v4
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s0, v5, 0
-; GFX7-NEXT: v_add_i32_e32 v4, vcc, s8, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, 0
-; GFX7-NEXT: v_mad_u64_u32 v[5:6], s[2:3], s1, v5, v[1:2]
-; GFX7-NEXT: v_mov_b32_e32 v1, s0
-; GFX7-NEXT: v_mad_u64_u32 v[3:4], s[2:3], s6, v1, v[3:4]
-; GFX7-NEXT: v_mov_b32_e32 v7, s5
-; GFX7-NEXT: v_mov_b32_e32 v1, v5
-; GFX7-NEXT: v_mad_u64_u32 v[1:2], s[2:3], s0, v7, v[1:2]
-; GFX7-NEXT: s_mul_i32 s7, s7, s0
-; GFX7-NEXT: v_add_i32_e32 v8, vcc, s7, v4
-; GFX7-NEXT: v_add_i32_e32 v4, vcc, v6, v2
-; GFX7-NEXT: v_addc_u32_e64 v5, s[2:3], 0, 0, vcc
-; GFX7-NEXT: s_mul_i32 s6, s6, s1
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s1, v7, v[4:5]
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, s6, v8
-; GFX7-NEXT: v_readfirstlane_b32 s0, v0
-; GFX7-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; GFX7-NEXT: v_addc_u32_e32 v2, vcc, v5, v2, vcc
-; GFX7-NEXT: v_readfirstlane_b32 s2, v3
-; GFX7-NEXT: v_readfirstlane_b32 s3, v2
-; GFX7-NEXT: v_readfirstlane_b32 s1, v1
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: v_mul_hi_u32 v2, s1, v1
+; GFX7-NEXT: s_mul_i32 s10, s0, s6
+; GFX7-NEXT: v_readfirstlane_b32 s9, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s6
+; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX7-NEXT: v_readfirstlane_b32 s13, v2
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: v_mul_hi_u32 v2, v2, s4
+; GFX7-NEXT: s_mul_i32 s12, s1, s5
+; GFX7-NEXT: v_readfirstlane_b32 s11, v0
+; GFX7-NEXT: s_add_u32 s10, s12, s10
+; GFX7-NEXT: v_mul_hi_u32 v1, s0, v1
+; GFX7-NEXT: v_mov_b32_e32 v0, s1
+; GFX7-NEXT: s_addc_u32 s11, s13, s11
+; GFX7-NEXT: s_mul_i32 s12, s2, s4
+; GFX7-NEXT: v_readfirstlane_b32 s13, v2
+; GFX7-NEXT: s_add_u32 s10, s12, s10
+; GFX7-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX7-NEXT: s_addc_u32 s11, s13, s11
+; GFX7-NEXT: s_mul_i32 s12, s0, s5
+; GFX7-NEXT: v_readfirstlane_b32 s13, v1
+; GFX7-NEXT: s_add_u32 s9, s12, s9
+; GFX7-NEXT: s_addc_u32 s10, s13, s10
+; GFX7-NEXT: s_mul_i32 s13, s1, s4
+; GFX7-NEXT: s_cselect_b32 s12, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s14, v0
+; GFX7-NEXT: s_add_u32 s9, s13, s9
+; GFX7-NEXT: s_mul_i32 s8, s0, s4
+; GFX7-NEXT: s_addc_u32 s10, s14, s10
+; GFX7-NEXT: s_mul_i32 s0, s0, s7
+; GFX7-NEXT: s_addc_u32 s0, s11, s0
+; GFX7-NEXT: s_mul_i32 s1, s1, s6
+; GFX7-NEXT: s_cmp_lg_u32 s12, 0
+; GFX7-NEXT: s_addc_u32 s0, s0, s1
+; GFX7-NEXT: s_mul_i32 s2, s2, s5
+; GFX7-NEXT: s_add_u32 s0, s2, s0
+; GFX7-NEXT: s_mul_i32 s3, s3, s4
+; GFX7-NEXT: s_add_u32 s3, s3, s0
+; GFX7-NEXT: s_mov_b32 s0, s8
+; GFX7-NEXT: s_mov_b32 s1, s9
+; GFX7-NEXT: s_mov_b32 s2, s10
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_mul_i128:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[8:9], s4, v0, 0
-; GFX8-NEXT: s_mul_i32 s3, s4, s3
-; GFX8-NEXT: v_mov_b32_e32 v5, s4
-; GFX8-NEXT: s_mul_i32 s8, s5, s2
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, s3, v4
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s0, v5, 0
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, s8, v2
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
-; GFX8-NEXT: v_mad_u64_u32 v[5:6], s[2:3], s1, v5, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
-; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[2:3], s6, v1, v[3:4]
-; GFX8-NEXT: v_mov_b32_e32 v7, s5
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
-; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[2:3], s0, v7, v[1:2]
-; GFX8-NEXT: s_mul_i32 s7, s7, s0
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, s7, v4
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v6, v2
-; GFX8-NEXT: v_addc_u32_e64 v5, s[2:3], 0, 0, vcc
-; GFX8-NEXT: s_mul_i32 s6, s6, s1
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s1, v7, v[4:5]
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v8
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v4, v3
-; GFX8-NEXT: v_addc_u32_e32 v2, vcc, v5, v2, vcc
-; GFX8-NEXT: v_readfirstlane_b32 s2, v3
-; GFX8-NEXT: v_readfirstlane_b32 s3, v2
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mul_hi_u32 v2, s1, v1
+; GFX8-NEXT: s_mul_i32 s10, s0, s6
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s13, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mul_hi_u32 v2, v2, s4
+; GFX8-NEXT: s_mul_i32 s12, s1, s5
+; GFX8-NEXT: v_readfirstlane_b32 s11, v0
+; GFX8-NEXT: s_add_u32 s10, s12, s10
+; GFX8-NEXT: v_mul_hi_u32 v1, s0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: s_addc_u32 s11, s13, s11
+; GFX8-NEXT: s_mul_i32 s12, s2, s4
+; GFX8-NEXT: v_readfirstlane_b32 s13, v2
+; GFX8-NEXT: s_add_u32 s10, s12, s10
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX8-NEXT: s_addc_u32 s11, s13, s11
+; GFX8-NEXT: s_mul_i32 s12, s0, s5
+; GFX8-NEXT: v_readfirstlane_b32 s13, v1
+; GFX8-NEXT: s_add_u32 s9, s12, s9
+; GFX8-NEXT: s_addc_u32 s10, s13, s10
+; GFX8-NEXT: s_mul_i32 s13, s1, s4
+; GFX8-NEXT: s_cselect_b32 s12, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s14, v0
+; GFX8-NEXT: s_add_u32 s9, s13, s9
+; GFX8-NEXT: s_mul_i32 s8, s0, s4
+; GFX8-NEXT: s_addc_u32 s10, s14, s10
+; GFX8-NEXT: s_mul_i32 s0, s0, s7
+; GFX8-NEXT: s_addc_u32 s0, s11, s0
+; GFX8-NEXT: s_mul_i32 s1, s1, s6
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_addc_u32 s0, s0, s1
+; GFX8-NEXT: s_mul_i32 s2, s2, s5
+; GFX8-NEXT: s_add_u32 s0, s2, s0
+; GFX8-NEXT: s_mul_i32 s3, s3, s4
+; GFX8-NEXT: s_add_u32 s3, s3, s0
+; GFX8-NEXT: s_mov_b32 s0, s8
+; GFX8-NEXT: s_mov_b32 s1, s9
+; GFX8-NEXT: s_mov_b32 s2, s10
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_mul_i128:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mul_i32 s3, s4, s3
-; GFX9-NEXT: s_mul_hi_u32 s8, s4, s2
-; GFX9-NEXT: s_add_i32 s3, s8, s3
-; GFX9-NEXT: s_mul_i32 s8, s5, s2
-; GFX9-NEXT: s_add_i32 s3, s3, s8
-; GFX9-NEXT: s_mul_i32 s8, s6, s1
-; GFX9-NEXT: s_mul_hi_u32 s9, s6, s0
-; GFX9-NEXT: s_add_i32 s8, s9, s8
-; GFX9-NEXT: s_mul_i32 s7, s7, s0
-; GFX9-NEXT: s_mul_i32 s2, s4, s2
-; GFX9-NEXT: s_add_i32 s8, s8, s7
-; GFX9-NEXT: s_mul_i32 s6, s6, s0
-; GFX9-NEXT: s_add_u32 s2, s6, s2
-; GFX9-NEXT: s_addc_u32 s3, s8, s3
-; GFX9-NEXT: s_mul_i32 s10, s1, s4
-; GFX9-NEXT: s_mul_hi_u32 s11, s0, s4
-; GFX9-NEXT: s_mul_hi_u32 s9, s1, s4
-; GFX9-NEXT: s_add_u32 s10, s10, s11
-; GFX9-NEXT: s_mul_i32 s7, s0, s5
-; GFX9-NEXT: s_addc_u32 s9, s9, 0
-; GFX9-NEXT: s_mul_hi_u32 s8, s0, s5
-; GFX9-NEXT: s_add_u32 s7, s7, s10
-; GFX9-NEXT: s_addc_u32 s8, s8, 0
-; GFX9-NEXT: s_add_u32 s8, s9, s8
-; GFX9-NEXT: s_addc_u32 s9, 0, 0
-; GFX9-NEXT: s_mul_hi_u32 s10, s1, s5
-; GFX9-NEXT: s_mul_i32 s1, s1, s5
-; GFX9-NEXT: s_add_u32 s1, s1, s8
-; GFX9-NEXT: s_mov_b32 s6, 0
-; GFX9-NEXT: s_addc_u32 s5, s10, s9
-; GFX9-NEXT: s_add_u32 s2, s1, s2
-; GFX9-NEXT: s_mul_i32 s0, s0, s4
-; GFX9-NEXT: s_mov_b32 s1, s6
-; GFX9-NEXT: s_addc_u32 s3, s5, s3
-; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX9-NEXT: s_mul_i32 s10, s0, s6
+; GFX9-NEXT: s_mul_i32 s12, s1, s5
+; GFX9-NEXT: s_mul_hi_u32 s11, s0, s6
+; GFX9-NEXT: s_mul_hi_u32 s13, s1, s5
+; GFX9-NEXT: s_add_u32 s10, s12, s10
+; GFX9-NEXT: s_addc_u32 s11, s13, s11
+; GFX9-NEXT: s_mul_i32 s12, s2, s4
+; GFX9-NEXT: s_mul_hi_u32 s13, s2, s4
+; GFX9-NEXT: s_add_u32 s10, s12, s10
+; GFX9-NEXT: s_mul_hi_u32 s9, s0, s4
+; GFX9-NEXT: s_addc_u32 s11, s13, s11
+; GFX9-NEXT: s_mul_i32 s12, s0, s5
+; GFX9-NEXT: s_mul_hi_u32 s13, s0, s5
+; GFX9-NEXT: s_add_u32 s9, s12, s9
+; GFX9-NEXT: s_addc_u32 s10, s13, s10
+; GFX9-NEXT: s_mul_i32 s13, s1, s4
+; GFX9-NEXT: s_cselect_b32 s12, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s14, s1, s4
+; GFX9-NEXT: s_add_u32 s9, s13, s9
+; GFX9-NEXT: s_mul_i32 s8, s0, s4
+; GFX9-NEXT: s_addc_u32 s10, s14, s10
+; GFX9-NEXT: s_mul_i32 s0, s0, s7
+; GFX9-NEXT: s_addc_u32 s0, s11, s0
+; GFX9-NEXT: s_mul_i32 s1, s1, s6
+; GFX9-NEXT: s_cmp_lg_u32 s12, 0
+; GFX9-NEXT: s_addc_u32 s0, s0, s1
+; GFX9-NEXT: s_mul_i32 s2, s2, s5
+; GFX9-NEXT: s_add_u32 s0, s2, s0
+; GFX9-NEXT: s_mul_i32 s3, s3, s4
+; GFX9-NEXT: s_add_u32 s3, s3, s0
+; GFX9-NEXT: s_mov_b32 s0, s8
+; GFX9-NEXT: s_mov_b32 s1, s9
+; GFX9-NEXT: s_mov_b32 s2, s10
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_mul_i128:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_mul_i32 s3, s4, s3
-; GFX10PLUS-NEXT: s_mul_hi_u32 s8, s4, s2
-; GFX10PLUS-NEXT: s_mul_hi_u32 s9, s6, s0
-; GFX10PLUS-NEXT: s_add_i32 s3, s8, s3
-; GFX10PLUS-NEXT: s_mul_i32 s8, s5, s2
-; GFX10PLUS-NEXT: s_mul_i32 s7, s7, s0
-; GFX10PLUS-NEXT: s_add_i32 s3, s3, s8
-; GFX10PLUS-NEXT: s_mul_i32 s8, s6, s1
-; GFX10PLUS-NEXT: s_mul_i32 s2, s4, s2
-; GFX10PLUS-NEXT: s_add_i32 s8, s9, s8
-; GFX10PLUS-NEXT: s_mul_i32 s6, s6, s0
-; GFX10PLUS-NEXT: s_add_i32 s8, s8, s7
-; GFX10PLUS-NEXT: s_add_u32 s2, s6, s2
-; GFX10PLUS-NEXT: s_addc_u32 s3, s8, s3
-; GFX10PLUS-NEXT: s_mul_i32 s8, s1, s4
-; GFX10PLUS-NEXT: s_mul_hi_u32 s9, s0, s4
-; GFX10PLUS-NEXT: s_mul_hi_u32 s10, s1, s4
-; GFX10PLUS-NEXT: s_mul_i32 s7, s0, s5
-; GFX10PLUS-NEXT: s_add_u32 s8, s8, s9
-; GFX10PLUS-NEXT: s_mul_hi_u32 s6, s0, s5
-; GFX10PLUS-NEXT: s_addc_u32 s9, s10, 0
-; GFX10PLUS-NEXT: s_add_u32 s7, s7, s8
-; GFX10PLUS-NEXT: s_addc_u32 s8, s6, 0
-; GFX10PLUS-NEXT: s_mul_i32 s10, s1, s5
-; GFX10PLUS-NEXT: s_add_u32 s8, s9, s8
-; GFX10PLUS-NEXT: s_mov_b32 s6, 0
-; GFX10PLUS-NEXT: s_addc_u32 s9, 0, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s1, s1, s5
-; GFX10PLUS-NEXT: s_add_u32 s5, s10, s8
-; GFX10PLUS-NEXT: s_addc_u32 s8, s1, s9
+; GFX10PLUS-NEXT: s_mul_i32 s9, s0, s6
+; GFX10PLUS-NEXT: s_mul_i32 s11, s1, s5
+; GFX10PLUS-NEXT: s_mul_hi_u32 s10, s0, s6
+; GFX10PLUS-NEXT: s_mul_hi_u32 s12, s1, s5
+; GFX10PLUS-NEXT: s_add_u32 s9, s11, s9
+; GFX10PLUS-NEXT: s_mul_i32 s11, s2, s4
+; GFX10PLUS-NEXT: s_addc_u32 s10, s12, s10
+; GFX10PLUS-NEXT: s_mul_hi_u32 s12, s2, s4
+; GFX10PLUS-NEXT: s_mul_hi_u32 s8, s0, s4
+; GFX10PLUS-NEXT: s_add_u32 s9, s11, s9
+; GFX10PLUS-NEXT: s_mul_i32 s11, s0, s5
+; GFX10PLUS-NEXT: s_addc_u32 s10, s12, s10
+; GFX10PLUS-NEXT: s_mul_hi_u32 s12, s0, s5
+; GFX10PLUS-NEXT: s_add_u32 s8, s11, s8
+; GFX10PLUS-NEXT: s_addc_u32 s9, s12, s9
+; GFX10PLUS-NEXT: s_mul_i32 s12, s1, s4
+; GFX10PLUS-NEXT: s_mul_hi_u32 s13, s1, s4
+; GFX10PLUS-NEXT: s_cselect_b32 s11, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s8, s12, s8
+; GFX10PLUS-NEXT: s_mul_i32 s12, s0, s7
+; GFX10PLUS-NEXT: s_addc_u32 s7, s13, s9
+; GFX10PLUS-NEXT: s_addc_u32 s9, s10, s12
+; GFX10PLUS-NEXT: s_mul_i32 s1, s1, s6
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10PLUS-NEXT: s_mul_i32 s2, s2, s5
+; GFX10PLUS-NEXT: s_addc_u32 s1, s9, s1
+; GFX10PLUS-NEXT: s_mul_i32 s3, s3, s4
+; GFX10PLUS-NEXT: s_add_i32 s1, s1, s2
; GFX10PLUS-NEXT: s_mul_i32 s0, s0, s4
-; GFX10PLUS-NEXT: s_mov_b32 s1, s6
-; GFX10PLUS-NEXT: s_add_u32 s2, s5, s2
-; GFX10PLUS-NEXT: s_addc_u32 s3, s8, s3
-; GFX10PLUS-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX10PLUS-NEXT: s_add_i32 s3, s1, s3
+; GFX10PLUS-NEXT: s_mov_b32 s1, s8
+; GFX10PLUS-NEXT: s_mov_b32 s2, s7
; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_mul_i128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mov_b32 s9, 0
-; GFX12-NEXT: s_mov_b32 s8, s4
-; GFX12-NEXT: s_mov_b32 s10, s0
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s12, s1
-; GFX12-NEXT: s_mul_u64 s[14:15], s[10:11], s[8:9]
-; GFX12-NEXT: s_mov_b32 s13, s9
-; GFX12-NEXT: s_mov_b32 s18, s5
-; GFX12-NEXT: s_mul_u64 s[16:17], s[12:13], s[8:9]
-; GFX12-NEXT: s_mov_b32 s8, s15
-; GFX12-NEXT: s_mov_b32 s19, s9
-; GFX12-NEXT: s_add_nc_u64 s[16:17], s[16:17], s[8:9]
-; GFX12-NEXT: s_mul_u64 s[10:11], s[10:11], s[18:19]
-; GFX12-NEXT: s_mov_b32 s8, s17
-; GFX12-NEXT: s_mov_b32 s17, s9
-; GFX12-NEXT: s_mul_u64 s[12:13], s[12:13], s[18:19]
-; GFX12-NEXT: s_add_nc_u64 s[10:11], s[10:11], s[16:17]
-; GFX12-NEXT: s_mul_u64 s[2:3], s[4:5], s[2:3]
-; GFX12-NEXT: s_mov_b32 s16, s11
-; GFX12-NEXT: s_mul_u64 s[0:1], s[6:7], s[0:1]
-; GFX12-NEXT: s_add_nc_u64 s[16:17], s[8:9], s[16:17]
-; GFX12-NEXT: s_mov_b32 s4, s9
-; GFX12-NEXT: s_add_nc_u64 s[12:13], s[12:13], s[16:17]
-; GFX12-NEXT: s_mov_b32 s5, s10
-; GFX12-NEXT: s_mov_b32 s15, s9
-; GFX12-NEXT: s_add_nc_u64 s[2:3], s[0:1], s[2:3]
-; GFX12-NEXT: s_or_b64 s[0:1], s[14:15], s[4:5]
-; GFX12-NEXT: s_add_nc_u64 s[2:3], s[12:13], s[2:3]
+; GFX12-NEXT: s_mul_i32 s9, s0, s6
+; GFX12-NEXT: s_mul_i32 s11, s1, s5
+; GFX12-NEXT: s_mul_hi_u32 s10, s0, s6
+; GFX12-NEXT: s_mul_hi_u32 s12, s1, s5
+; GFX12-NEXT: s_add_co_u32 s9, s11, s9
+; GFX12-NEXT: s_mul_i32 s11, s2, s4
+; GFX12-NEXT: s_add_co_ci_u32 s10, s12, s10
+; GFX12-NEXT: s_mul_hi_u32 s12, s2, s4
+; GFX12-NEXT: s_mul_hi_u32 s8, s0, s4
+; GFX12-NEXT: s_add_co_u32 s9, s11, s9
+; GFX12-NEXT: s_mul_i32 s11, s0, s5
+; GFX12-NEXT: s_add_co_ci_u32 s10, s12, s10
+; GFX12-NEXT: s_mul_hi_u32 s12, s0, s5
+; GFX12-NEXT: s_add_co_u32 s8, s11, s8
+; GFX12-NEXT: s_add_co_ci_u32 s9, s12, s9
+; GFX12-NEXT: s_mul_i32 s12, s1, s4
+; GFX12-NEXT: s_mul_hi_u32 s13, s1, s4
+; GFX12-NEXT: s_cselect_b32 s11, 1, 0
+; GFX12-NEXT: s_add_co_u32 s8, s12, s8
+; GFX12-NEXT: s_mul_i32 s12, s0, s7
+; GFX12-NEXT: s_add_co_ci_u32 s7, s13, s9
+; GFX12-NEXT: s_add_co_ci_u32 s9, s10, s12
+; GFX12-NEXT: s_mul_i32 s1, s1, s6
+; GFX12-NEXT: s_cmp_lg_u32 s11, 0
+; GFX12-NEXT: s_mul_i32 s2, s2, s5
+; GFX12-NEXT: s_add_co_ci_u32 s1, s9, s1
+; GFX12-NEXT: s_mul_i32 s3, s3, s4
+; GFX12-NEXT: s_add_co_i32 s1, s1, s2
+; GFX12-NEXT: s_mul_i32 s0, s0, s4
+; GFX12-NEXT: s_add_co_i32 s3, s1, s3
+; GFX12-NEXT: s_mov_b32 s1, s8
+; GFX12-NEXT: s_mov_b32 s2, s7
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: s_mul_i128:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[10:11], 0xffffffff
-; GFX1250-NEXT: s_mov_b32 s9, 0
-; GFX1250-NEXT: s_mov_b32 s8, s4
-; GFX1250-NEXT: s_and_b64 s[10:11], s[0:1], s[10:11]
-; GFX1250-NEXT: s_mov_b32 s12, s1
-; GFX1250-NEXT: s_mov_b32 s13, s9
-; GFX1250-NEXT: s_mul_u64 s[14:15], s[10:11], s[8:9]
-; GFX1250-NEXT: s_mul_u64 s[16:17], s[12:13], s[8:9]
-; GFX1250-NEXT: s_mov_b32 s8, s15
-; GFX1250-NEXT: s_mov_b32 s18, s5
-; GFX1250-NEXT: s_add_nc_u64 s[16:17], s[16:17], s[8:9]
-; GFX1250-NEXT: s_mov_b32 s19, s9
-; GFX1250-NEXT: s_mov_b32 s8, s17
-; GFX1250-NEXT: s_mov_b32 s17, s9
-; GFX1250-NEXT: s_mul_u64 s[10:11], s[10:11], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[12:13], s[12:13], s[18:19]
-; GFX1250-NEXT: s_add_nc_u64 s[10:11], s[10:11], s[16:17]
-; GFX1250-NEXT: s_mul_u64 s[2:3], s[4:5], s[2:3]
-; GFX1250-NEXT: s_mov_b32 s16, s11
-; GFX1250-NEXT: s_mul_u64 s[0:1], s[6:7], s[0:1]
-; GFX1250-NEXT: s_add_nc_u64 s[16:17], s[8:9], s[16:17]
-; GFX1250-NEXT: s_mov_b32 s4, s9
-; GFX1250-NEXT: s_add_nc_u64 s[12:13], s[12:13], s[16:17]
-; GFX1250-NEXT: s_mov_b32 s5, s10
-; GFX1250-NEXT: s_mov_b32 s15, s9
-; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[0:1], s[2:3]
-; GFX1250-NEXT: s_or_b64 s[0:1], s[14:15], s[4:5]
-; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[12:13], s[2:3]
+; GFX1250-NEXT: s_mul_i32 s9, s0, s6
+; GFX1250-NEXT: s_mul_i32 s11, s1, s5
+; GFX1250-NEXT: s_mul_hi_u32 s10, s0, s6
+; GFX1250-NEXT: s_mul_hi_u32 s12, s1, s5
+; GFX1250-NEXT: s_add_co_u32 s9, s11, s9
+; GFX1250-NEXT: s_mul_i32 s11, s2, s4
+; GFX1250-NEXT: s_add_co_ci_u32 s10, s12, s10
+; GFX1250-NEXT: s_mul_hi_u32 s12, s2, s4
+; GFX1250-NEXT: s_mul_hi_u32 s8, s0, s4
+; GFX1250-NEXT: s_add_co_u32 s9, s11, s9
+; GFX1250-NEXT: s_mul_i32 s11, s0, s5
+; GFX1250-NEXT: s_add_co_ci_u32 s10, s12, s10
+; GFX1250-NEXT: s_mul_hi_u32 s12, s0, s5
+; GFX1250-NEXT: s_add_co_u32 s8, s11, s8
+; GFX1250-NEXT: s_add_co_ci_u32 s9, s12, s9
+; GFX1250-NEXT: s_mul_i32 s12, s1, s4
+; GFX1250-NEXT: s_mul_hi_u32 s13, s1, s4
+; GFX1250-NEXT: s_cselect_b32 s11, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s8, s12, s8
+; GFX1250-NEXT: s_mul_i32 s12, s0, s7
+; GFX1250-NEXT: s_add_co_ci_u32 s7, s13, s9
+; GFX1250-NEXT: s_add_co_ci_u32 s9, s10, s12
+; GFX1250-NEXT: s_mul_i32 s1, s1, s6
+; GFX1250-NEXT: s_cmp_lg_u32 s11, 0
+; GFX1250-NEXT: s_mul_i32 s2, s2, s5
+; GFX1250-NEXT: s_add_co_ci_u32 s1, s9, s1
+; GFX1250-NEXT: s_mul_i32 s3, s3, s4
+; GFX1250-NEXT: s_add_co_i32 s1, s1, s2
+; GFX1250-NEXT: s_mul_i32 s0, s0, s4
+; GFX1250-NEXT: s_add_co_i32 s3, s1, s3
+; GFX1250-NEXT: s_mov_b32 s1, s8
+; GFX1250-NEXT: s_mov_b32 s2, s7
; GFX1250-NEXT: ; return to shader part epilog
;
; GFX13-LABEL: s_mul_i128:
; GFX13: ; %bb.0:
-; GFX13-NEXT: s_mov_b64 s[10:11], 0xffffffff
-; GFX13-NEXT: s_mov_b32 s9, 0
-; GFX13-NEXT: s_mov_b32 s8, s4
-; GFX13-NEXT: s_and_b64 s[10:11], s[0:1], s[10:11]
-; GFX13-NEXT: s_mov_b32 s12, s1
-; GFX13-NEXT: s_mul_u64 s[14:15], s[10:11], s[8:9]
-; GFX13-NEXT: s_mov_b32 s13, s9
-; GFX13-NEXT: s_mov_b32 s18, s5
-; GFX13-NEXT: s_mul_u64 s[16:17], s[12:13], s[8:9]
-; GFX13-NEXT: s_mov_b32 s8, s15
-; GFX13-NEXT: s_mov_b32 s19, s9
-; GFX13-NEXT: s_add_nc_u64 s[16:17], s[16:17], s[8:9]
-; GFX13-NEXT: s_mul_u64 s[10:11], s[10:11], s[18:19]
-; GFX13-NEXT: s_mov_b32 s8, s17
-; GFX13-NEXT: s_mov_b32 s17, s9
-; GFX13-NEXT: s_mul_u64 s[12:13], s[12:13], s[18:19]
-; GFX13-NEXT: s_add_nc_u64 s[10:11], s[10:11], s[16:17]
-; GFX13-NEXT: s_mul_u64 s[2:3], s[4:5], s[2:3]
-; GFX13-NEXT: s_mov_b32 s16, s11
-; GFX13-NEXT: s_mul_u64 s[0:1], s[6:7], s[0:1]
-; GFX13-NEXT: s_add_nc_u64 s[16:17], s[8:9], s[16:17]
-; GFX13-NEXT: s_mov_b32 s4, s9
-; GFX13-NEXT: s_add_nc_u64 s[12:13], s[12:13], s[16:17]
-; GFX13-NEXT: s_mov_b32 s5, s10
-; GFX13-NEXT: s_mov_b32 s15, s9
-; GFX13-NEXT: s_add_nc_u64 s[2:3], s[0:1], s[2:3]
-; GFX13-NEXT: s_or_b64 s[0:1], s[14:15], s[4:5]
-; GFX13-NEXT: s_add_nc_u64 s[2:3], s[12:13], s[2:3]
+; GFX13-NEXT: s_mul_i32 s9, s0, s6
+; GFX13-NEXT: s_mul_i32 s11, s1, s5
+; GFX13-NEXT: s_mul_hi_u32 s10, s0, s6
+; GFX13-NEXT: s_mul_hi_u32 s12, s1, s5
+; GFX13-NEXT: s_add_co_u32 s9, s11, s9
+; GFX13-NEXT: s_mul_i32 s11, s2, s4
+; GFX13-NEXT: s_add_co_ci_u32 s10, s12, s10
+; GFX13-NEXT: s_mul_hi_u32 s12, s2, s4
+; GFX13-NEXT: s_mul_hi_u32 s8, s0, s4
+; GFX13-NEXT: s_add_co_u32 s9, s11, s9
+; GFX13-NEXT: s_mul_i32 s11, s0, s5
+; GFX13-NEXT: s_add_co_ci_u32 s10, s12, s10
+; GFX13-NEXT: s_mul_hi_u32 s12, s0, s5
+; GFX13-NEXT: s_add_co_u32 s8, s11, s8
+; GFX13-NEXT: s_add_co_ci_u32 s9, s12, s9
+; GFX13-NEXT: s_mul_i32 s12, s1, s4
+; GFX13-NEXT: s_mul_hi_u32 s13, s1, s4
+; GFX13-NEXT: s_cselect_b32 s11, 1, 0
+; GFX13-NEXT: s_add_co_u32 s8, s12, s8
+; GFX13-NEXT: s_mul_i32 s12, s0, s7
+; GFX13-NEXT: s_add_co_ci_u32 s7, s13, s9
+; GFX13-NEXT: s_add_co_ci_u32 s9, s10, s12
+; GFX13-NEXT: s_mul_i32 s1, s1, s6
+; GFX13-NEXT: s_cmp_lg_u32 s11, 0
+; GFX13-NEXT: s_mul_i32 s2, s2, s5
+; GFX13-NEXT: s_add_co_ci_u32 s1, s9, s1
+; GFX13-NEXT: s_mul_i32 s3, s3, s4
+; GFX13-NEXT: s_add_co_i32 s1, s1, s2
+; GFX13-NEXT: s_mul_i32 s0, s0, s4
+; GFX13-NEXT: s_add_co_i32 s3, s1, s3
+; GFX13-NEXT: s_mov_b32 s1, s8
+; GFX13-NEXT: s_mov_b32 s2, s7
; GFX13-NEXT: ; return to shader part epilog
%result = mul i128 %num, %den
%cast = bitcast i128 %result to <4 x i32>
@@ -1578,1139 +1617,1397 @@ define i128 @v_mul_i128(i128 %num, i128 %den) {
define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
; GFX7-LABEL: s_mul_i256:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_mov_b32_e32 v15, s8
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[16:17], s2, v15, 0
-; GFX7-NEXT: v_mad_u64_u32 v[3:4], s[16:17], s0, v15, 0
-; GFX7-NEXT: v_mov_b32_e32 v2, 0
-; GFX7-NEXT: v_mov_b32_e32 v5, v2
-; GFX7-NEXT: v_mad_u64_u32 v[6:7], s[16:17], s3, v15, v[1:2]
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s1, v15, v[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v16, s10
-; GFX7-NEXT: v_mov_b32_e32 v17, s11
-; GFX7-NEXT: v_mov_b32_e32 v1, v6
-; GFX7-NEXT: v_mov_b32_e32 v6, s9
-; GFX7-NEXT: v_mad_u64_u32 v[8:9], s[16:17], s2, v6, v[1:2]
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mad_u64_u32 v[10:11], s[16:17], s0, v6, v[1:2]
-; GFX7-NEXT: v_mov_b32_e32 v22, s1
-; GFX7-NEXT: s_mul_i32 s11, s4, s11
-; GFX7-NEXT: s_mul_i32 s10, s5, s10
-; GFX7-NEXT: v_add_i32_e32 v4, vcc, v5, v11
-; GFX7-NEXT: v_addc_u32_e64 v5, s[16:17], 0, 0, vcc
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s1, v6, v[4:5]
-; GFX7-NEXT: s_mul_i32 s7, s7, s8
-; GFX7-NEXT: v_add_i32_e32 v13, vcc, v0, v4
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[16:17], s0, v16, 0
-; GFX7-NEXT: v_addc_u32_e32 v8, vcc, v8, v5, vcc
-; GFX7-NEXT: v_mad_u64_u32 v[11:12], s[16:17], s1, v16, v[1:2]
-; GFX7-NEXT: v_mov_b32_e32 v1, v11
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s0, v17, v[1:2]
-; GFX7-NEXT: v_add_i32_e64 v18, s[16:17], v0, v13
-; GFX7-NEXT: v_add_i32_e64 v5, s[18:19], v12, v5
-; GFX7-NEXT: v_addc_u32_e64 v19, s[16:17], v4, v8, s[16:17]
-; GFX7-NEXT: v_addc_u32_e64 v4, s[18:19], 0, 0, s[18:19]
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[18:19], s1, v17, 0
-; GFX7-NEXT: s_mul_i32 s1, s1, s14
-; GFX7-NEXT: v_addc_u32_e64 v8, s[16:17], v0, v5, s[16:17]
-; GFX7-NEXT: v_addc_u32_e64 v11, s[16:17], v1, v4, s[16:17]
-; GFX7-NEXT: v_add_i32_e64 v7, s[16:17], v7, v9
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[18:19], s3, v6, 0
-; GFX7-NEXT: v_addc_u32_e64 v6, s[16:17], 0, 0, s[16:17]
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[16:17], s2, v16, 0
-; GFX7-NEXT: v_addc_u32_e32 v4, vcc, v4, v7, vcc
-; GFX7-NEXT: v_addc_u32_e32 v5, vcc, v5, v6, vcc
-; GFX7-NEXT: v_mad_u64_u32 v[6:7], s[16:17], s3, v16, v[1:2]
-; GFX7-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; GFX7-NEXT: v_addc_u32_e32 v20, vcc, v5, v11, vcc
-; GFX7-NEXT: v_mov_b32_e32 v1, v6
-; GFX7-NEXT: v_mad_u64_u32 v[8:9], s[16:17], s2, v17, v[1:2]
-; GFX7-NEXT: v_add_i32_e64 v21, s[16:17], v0, v4
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[18:19], s8, v4, 0
-; GFX7-NEXT: v_mov_b32_e32 v6, v2
-; GFX7-NEXT: v_addc_u32_e64 v8, s[20:21], v8, v20, s[16:17]
-; GFX7-NEXT: v_mad_u64_u32 v[11:12], s[18:19], s9, v4, v[1:2]
-; GFX7-NEXT: v_mov_b32_e32 v1, v11
-; GFX7-NEXT: v_mov_b32_e32 v11, s0
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[18:19], s12, v11, 0
-; GFX7-NEXT: v_mad_u64_u32 v[5:6], s[18:19], s13, v11, v[5:6]
-; GFX7-NEXT: v_mov_b32_e32 v11, s5
-; GFX7-NEXT: v_mad_u64_u32 v[13:14], s[18:19], s8, v11, v[1:2]
-; GFX7-NEXT: v_mov_b32_e32 v1, v5
-; GFX7-NEXT: v_mad_u64_u32 v[1:2], s[18:19], s12, v22, v[1:2]
-; GFX7-NEXT: v_add_i32_e64 v23, s[18:19], v4, v0
-; GFX7-NEXT: v_add_i32_e64 v4, s[16:17], v7, v9
-; GFX7-NEXT: v_addc_u32_e64 v5, s[16:17], 0, 0, s[16:17]
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s3, v17, v[4:5]
-; GFX7-NEXT: s_and_b64 s[16:17], vcc, exec
-; GFX7-NEXT: s_cselect_b64 s[16:17], 1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s16
-; GFX7-NEXT: v_mov_b32_e32 v7, s17
-; GFX7-NEXT: v_addc_u32_e64 v9, vcc, v4, v0, s[20:21]
-; GFX7-NEXT: v_addc_u32_e32 v7, vcc, v5, v7, vcc
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s4, v16, 0
-; GFX7-NEXT: v_add_i32_e64 v0, s[20:21], v12, v14
-; GFX7-NEXT: v_addc_u32_e64 v1, s[18:19], v1, v13, s[18:19]
-; GFX7-NEXT: v_add_i32_e32 v12, vcc, v21, v23
-; GFX7-NEXT: v_addc_u32_e64 v8, s[16:17], v8, v1, vcc
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s11, v5
-; GFX7-NEXT: v_add_i32_e32 v5, vcc, s10, v1
-; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s6, v15, v[4:5]
-; GFX7-NEXT: v_addc_u32_e64 v1, s[4:5], 0, 0, s[20:21]
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s9, v11, v[0:1]
-; GFX7-NEXT: s_mul_i32 s6, s6, s9
-; GFX7-NEXT: v_add_i32_e32 v5, vcc, s7, v5
-; GFX7-NEXT: v_add_i32_e32 v5, vcc, s6, v5
-; GFX7-NEXT: v_add_i32_e32 v11, vcc, v0, v4
+; GFX7-NEXT: s_mov_b32 s16, s0
+; GFX7-NEXT: v_mov_b32_e32 v0, s8
+; GFX7-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, s9
+; GFX7-NEXT: v_mul_hi_u32 v2, s1, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, s16, v1
+; GFX7-NEXT: v_readfirstlane_b32 s17, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s10
+; GFX7-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX7-NEXT: v_readfirstlane_b32 s21, v2
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: v_mul_hi_u32 v3, v2, s8
+; GFX7-NEXT: s_mul_i32 s18, s16, s10
+; GFX7-NEXT: s_mul_i32 s20, s1, s9
+; GFX7-NEXT: v_readfirstlane_b32 s19, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s1
+; GFX7-NEXT: s_add_u32 s18, s20, s18
+; GFX7-NEXT: s_addc_u32 s19, s21, s19
+; GFX7-NEXT: s_mul_i32 s21, s2, s8
+; GFX7-NEXT: v_readfirstlane_b32 s23, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, v0, s8
+; GFX7-NEXT: s_cselect_b32 s20, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s22, v3
+; GFX7-NEXT: s_add_u32 s18, s21, s18
+; GFX7-NEXT: s_addc_u32 s19, s22, s19
+; GFX7-NEXT: s_mul_i32 s22, s16, s9
+; GFX7-NEXT: s_cselect_b32 s21, 1, 0
+; GFX7-NEXT: s_add_u32 s17, s22, s17
+; GFX7-NEXT: s_addc_u32 s22, s23, s18
+; GFX7-NEXT: v_readfirstlane_b32 s23, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, s12
+; GFX7-NEXT: v_mul_hi_u32 v3, s16, v1
+; GFX7-NEXT: s_mul_i32 s18, s1, s8
+; GFX7-NEXT: s_cselect_b32 s25, 1, 0
+; GFX7-NEXT: s_add_u32 s18, s18, s17
+; GFX7-NEXT: s_addc_u32 s17, s23, s22
+; GFX7-NEXT: v_mov_b32_e32 v4, s11
+; GFX7-NEXT: v_readfirstlane_b32 s23, v3
+; GFX7-NEXT: v_mul_hi_u32 v3, v2, s10
+; GFX7-NEXT: v_mul_hi_u32 v5, s1, v4
+; GFX7-NEXT: s_mul_i32 s22, s16, s12
+; GFX7-NEXT: s_mul_i32 s24, s1, s11
+; GFX7-NEXT: v_readfirstlane_b32 s28, v3
+; GFX7-NEXT: v_mov_b32_e32 v3, s3
+; GFX7-NEXT: v_readfirstlane_b32 s27, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v3, s9
+; GFX7-NEXT: s_cselect_b32 s26, 1, 0
+; GFX7-NEXT: s_add_u32 s24, s24, s22
+; GFX7-NEXT: s_addc_u32 s23, s27, s23
+; GFX7-NEXT: v_readfirstlane_b32 s29, v5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
+; GFX7-NEXT: v_mul_hi_u32 v6, v5, s8
+; GFX7-NEXT: s_mul_i32 s27, s2, s10
+; GFX7-NEXT: s_cselect_b32 s22, 1, 0
+; GFX7-NEXT: s_add_u32 s24, s27, s24
+; GFX7-NEXT: v_mul_hi_u32 v0, v0, s10
+; GFX7-NEXT: s_addc_u32 s27, s28, s23
+; GFX7-NEXT: s_mul_i32 s28, s3, s9
+; GFX7-NEXT: s_cselect_b32 s23, 1, 0
+; GFX7-NEXT: s_add_u32 s28, s28, s24
+; GFX7-NEXT: v_readfirstlane_b32 s30, v6
+; GFX7-NEXT: v_mul_hi_u32 v6, s16, v4
+; GFX7-NEXT: s_addc_u32 s27, s29, s27
+; GFX7-NEXT: s_mul_i32 s29, s4, s8
+; GFX7-NEXT: s_cselect_b32 s24, 1, 0
+; GFX7-NEXT: s_add_u32 s28, s29, s28
+; GFX7-NEXT: v_readfirstlane_b32 s33, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, v2, s9
+; GFX7-NEXT: s_addc_u32 s27, s30, s27
+; GFX7-NEXT: s_mul_i32 s30, s16, s11
+; GFX7-NEXT: s_cselect_b32 s29, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s31, v6
+; GFX7-NEXT: s_add_u32 s19, s30, s19
+; GFX7-NEXT: s_addc_u32 s28, s31, s28
+; GFX7-NEXT: s_mul_i32 s31, s1, s10
+; GFX7-NEXT: s_cselect_b32 s30, 1, 0
+; GFX7-NEXT: s_add_u32 s19, s31, s19
+; GFX7-NEXT: v_readfirstlane_b32 s34, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, v3, s8
+; GFX7-NEXT: s_addc_u32 s28, s33, s28
+; GFX7-NEXT: s_mul_i32 s33, s2, s9
+; GFX7-NEXT: s_cselect_b32 s31, 1, 0
+; GFX7-NEXT: s_add_u32 s19, s33, s19
+; GFX7-NEXT: s_addc_u32 s28, s34, s28
+; GFX7-NEXT: s_mul_i32 s34, s3, s8
+; GFX7-NEXT: s_cselect_b32 s33, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s35, v0
+; GFX7-NEXT: s_add_u32 s19, s34, s19
+; GFX7-NEXT: s_addc_u32 s28, s35, s28
+; GFX7-NEXT: s_cselect_b32 s34, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s25, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s14
-; GFX7-NEXT: v_addc_u32_e32 v13, vcc, v1, v5, vcc
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s0, v0, 0
-; GFX7-NEXT: s_mul_i32 s0, s0, s15
-; GFX7-NEXT: v_mov_b32_e32 v4, s12
-; GFX7-NEXT: s_mul_i32 s3, s3, s12
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s0, v1
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s1, v1
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v4, v[0:1]
-; GFX7-NEXT: v_add_i32_e64 v4, s[0:1], v6, v2
-; GFX7-NEXT: v_addc_u32_e64 v5, s[0:1], 0, 0, s[0:1]
-; GFX7-NEXT: v_add_i32_e32 v6, vcc, s3, v1
-; GFX7-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s13, v22, v[4:5]
+; GFX7-NEXT: s_cselect_b32 s25, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s26, 0
+; GFX7-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX7-NEXT: s_addc_u32 s19, s25, s19
+; GFX7-NEXT: s_cselect_b32 s25, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s20, 0
+; GFX7-NEXT: v_mov_b32_e32 v2, s13
+; GFX7-NEXT: s_cselect_b32 s20, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s21, 0
+; GFX7-NEXT: v_mul_hi_u32 v6, s1, v2
+; GFX7-NEXT: s_addc_u32 s20, s20, 0
+; GFX7-NEXT: v_readfirstlane_b32 s26, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, s2, v1
+; GFX7-NEXT: s_cmp_lg_u32 s25, 0
+; GFX7-NEXT: s_addc_u32 s20, s20, s28
+; GFX7-NEXT: s_mul_i32 s25, s16, s14
+; GFX7-NEXT: s_mul_i32 s28, s1, s13
+; GFX7-NEXT: s_cselect_b32 s21, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s35, v6
+; GFX7-NEXT: s_add_u32 s25, s28, s25
+; GFX7-NEXT: s_addc_u32 s26, s35, s26
+; GFX7-NEXT: v_readfirstlane_b32 s35, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, v3, s11
+; GFX7-NEXT: s_mul_i32 s28, s2, s12
+; GFX7-NEXT: s_add_u32 s25, s28, s25
+; GFX7-NEXT: s_addc_u32 s26, s35, s26
+; GFX7-NEXT: v_readfirstlane_b32 s35, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, v5, s10
+; GFX7-NEXT: s_mul_i32 s28, s3, s11
+; GFX7-NEXT: s_add_u32 s25, s28, s25
+; GFX7-NEXT: s_addc_u32 s26, s35, s26
+; GFX7-NEXT: v_readfirstlane_b32 s35, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-NEXT: v_mul_hi_u32 v6, v0, s9
+; GFX7-NEXT: s_mul_i32 s28, s4, s10
+; GFX7-NEXT: s_add_u32 s25, s28, s25
+; GFX7-NEXT: v_mul_hi_u32 v1, s1, v1
+; GFX7-NEXT: s_addc_u32 s26, s35, s26
+; GFX7-NEXT: v_readfirstlane_b32 s35, v6
+; GFX7-NEXT: v_mov_b32_e32 v6, s6
+; GFX7-NEXT: v_mul_hi_u32 v6, v6, s8
+; GFX7-NEXT: s_mul_i32 s28, s5, s9
+; GFX7-NEXT: s_add_u32 s25, s28, s25
+; GFX7-NEXT: v_mul_hi_u32 v2, s16, v2
+; GFX7-NEXT: v_readfirstlane_b32 s36, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, s2, v4
+; GFX7-NEXT: s_addc_u32 s26, s35, s26
+; GFX7-NEXT: s_mul_i32 s28, s6, s8
+; GFX7-NEXT: v_readfirstlane_b32 s35, v6
+; GFX7-NEXT: s_add_u32 s25, s28, s25
+; GFX7-NEXT: s_addc_u32 s26, s35, s26
+; GFX7-NEXT: s_mul_i32 s28, s16, s13
+; GFX7-NEXT: v_readfirstlane_b32 s35, v2
+; GFX7-NEXT: s_add_u32 s27, s28, s27
+; GFX7-NEXT: v_readfirstlane_b32 s37, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, v3, s10
+; GFX7-NEXT: s_addc_u32 s25, s35, s25
+; GFX7-NEXT: s_mul_i32 s35, s1, s12
+; GFX7-NEXT: s_cselect_b32 s28, 1, 0
+; GFX7-NEXT: s_add_u32 s27, s35, s27
+; GFX7-NEXT: s_addc_u32 s25, s36, s25
+; GFX7-NEXT: s_mul_i32 s36, s2, s11
+; GFX7-NEXT: s_cselect_b32 s35, 1, 0
+; GFX7-NEXT: s_add_u32 s27, s36, s27
+; GFX7-NEXT: v_readfirstlane_b32 s38, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, v5, s9
+; GFX7-NEXT: s_addc_u32 s25, s37, s25
+; GFX7-NEXT: s_mul_i32 s37, s3, s10
+; GFX7-NEXT: s_cselect_b32 s36, 1, 0
+; GFX7-NEXT: s_add_u32 s27, s37, s27
+; GFX7-NEXT: v_mul_hi_u32 v0, v0, s8
+; GFX7-NEXT: s_addc_u32 s25, s38, s25
+; GFX7-NEXT: s_mul_i32 s38, s4, s9
+; GFX7-NEXT: s_cselect_b32 s37, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s39, v1
+; GFX7-NEXT: s_add_u32 s27, s38, s27
+; GFX7-NEXT: s_addc_u32 s25, s39, s25
+; GFX7-NEXT: s_mul_i32 s39, s5, s8
+; GFX7-NEXT: s_cselect_b32 s38, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s40, v0
+; GFX7-NEXT: s_add_u32 s27, s39, s27
+; GFX7-NEXT: s_addc_u32 s25, s40, s25
+; GFX7-NEXT: s_cselect_b32 s39, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s30, 0
+; GFX7-NEXT: s_cselect_b32 s30, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s31, 0
+; GFX7-NEXT: s_addc_u32 s30, s30, 0
+; GFX7-NEXT: s_cmp_lg_u32 s33, 0
+; GFX7-NEXT: s_addc_u32 s30, s30, 0
+; GFX7-NEXT: s_cmp_lg_u32 s34, 0
+; GFX7-NEXT: s_addc_u32 s30, s30, 0
+; GFX7-NEXT: s_cmp_lg_u32 s21, 0
+; GFX7-NEXT: s_addc_u32 s21, s30, s27
+; GFX7-NEXT: s_cselect_b32 s27, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s22, 0
+; GFX7-NEXT: s_cselect_b32 s22, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s23, 0
+; GFX7-NEXT: s_addc_u32 s22, s22, 0
+; GFX7-NEXT: s_cmp_lg_u32 s24, 0
+; GFX7-NEXT: s_addc_u32 s22, s22, 0
+; GFX7-NEXT: s_cmp_lg_u32 s29, 0
+; GFX7-NEXT: s_addc_u32 s22, s22, 0
+; GFX7-NEXT: s_cmp_lg_u32 s27, 0
+; GFX7-NEXT: s_addc_u32 s22, s22, s25
+; GFX7-NEXT: s_mul_i32 s16, s16, s15
+; GFX7-NEXT: s_addc_u32 s15, s26, s16
+; GFX7-NEXT: s_mul_i32 s1, s1, s14
+; GFX7-NEXT: s_cmp_lg_u32 s39, 0
+; GFX7-NEXT: s_addc_u32 s1, s15, s1
; GFX7-NEXT: s_mul_i32 s2, s2, s13
-; GFX7-NEXT: v_add_i32_e32 v4, vcc, s2, v6
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, v2, v4, vcc
-; GFX7-NEXT: v_addc_u32_e64 v0, vcc, v0, v11, s[18:19]
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, v1, v13, vcc
-; GFX7-NEXT: v_addc_u32_e64 v0, vcc, v9, v0, s[16:17]
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GFX7-NEXT: v_readfirstlane_b32 s6, v0
-; GFX7-NEXT: v_readfirstlane_b32 s7, v1
-; GFX7-NEXT: v_readfirstlane_b32 s4, v12
-; GFX7-NEXT: v_readfirstlane_b32 s5, v8
-; GFX7-NEXT: v_readfirstlane_b32 s2, v18
-; GFX7-NEXT: v_readfirstlane_b32 s3, v19
-; GFX7-NEXT: v_readfirstlane_b32 s0, v3
-; GFX7-NEXT: v_readfirstlane_b32 s1, v10
+; GFX7-NEXT: s_cmp_lg_u32 s38, 0
+; GFX7-NEXT: s_addc_u32 s1, s1, s2
+; GFX7-NEXT: s_mul_i32 s3, s3, s12
+; GFX7-NEXT: s_cmp_lg_u32 s37, 0
+; GFX7-NEXT: s_addc_u32 s1, s1, s3
+; GFX7-NEXT: s_mul_i32 s4, s4, s11
+; GFX7-NEXT: s_cmp_lg_u32 s36, 0
+; GFX7-NEXT: s_addc_u32 s1, s1, s4
+; GFX7-NEXT: s_mul_i32 s5, s5, s10
+; GFX7-NEXT: s_cmp_lg_u32 s35, 0
+; GFX7-NEXT: s_addc_u32 s1, s1, s5
+; GFX7-NEXT: s_mul_i32 s6, s6, s9
+; GFX7-NEXT: s_cmp_lg_u32 s28, 0
+; GFX7-NEXT: s_addc_u32 s1, s1, s6
+; GFX7-NEXT: s_mul_i32 s7, s7, s8
+; GFX7-NEXT: s_mul_i32 s0, s0, s8
+; GFX7-NEXT: s_add_u32 s7, s7, s1
+; GFX7-NEXT: s_mov_b32 s1, s18
+; GFX7-NEXT: s_mov_b32 s2, s17
+; GFX7-NEXT: s_mov_b32 s3, s19
+; GFX7-NEXT: s_mov_b32 s4, s20
+; GFX7-NEXT: s_mov_b32 s5, s21
+; GFX7-NEXT: s_mov_b32 s6, s22
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_mul_i256:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v15, s8
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[16:17], s2, v15, 0
-; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[16:17], s0, v15, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
-; GFX8-NEXT: v_mov_b32_e32 v5, v2
-; GFX8-NEXT: v_mad_u64_u32 v[6:7], s[16:17], s3, v15, v[1:2]
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s1, v15, v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v16, s10
-; GFX8-NEXT: v_mov_b32_e32 v17, s11
-; GFX8-NEXT: v_mov_b32_e32 v1, v6
-; GFX8-NEXT: v_mov_b32_e32 v6, s9
-; GFX8-NEXT: v_mad_u64_u32 v[8:9], s[16:17], s2, v6, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: v_mad_u64_u32 v[10:11], s[16:17], s0, v6, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v22, s1
-; GFX8-NEXT: s_mul_i32 s11, s4, s11
-; GFX8-NEXT: s_mul_i32 s10, s5, s10
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v5, v11
-; GFX8-NEXT: v_addc_u32_e64 v5, s[16:17], 0, 0, vcc
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s1, v6, v[4:5]
-; GFX8-NEXT: s_mul_i32 s7, s7, s8
-; GFX8-NEXT: v_add_u32_e32 v13, vcc, v0, v4
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[16:17], s0, v16, 0
-; GFX8-NEXT: v_addc_u32_e32 v8, vcc, v8, v5, vcc
-; GFX8-NEXT: v_mad_u64_u32 v[11:12], s[16:17], s1, v16, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v1, v11
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s0, v17, v[1:2]
-; GFX8-NEXT: v_add_u32_e64 v18, s[16:17], v0, v13
-; GFX8-NEXT: v_add_u32_e64 v5, s[18:19], v12, v5
-; GFX8-NEXT: v_addc_u32_e64 v19, s[16:17], v4, v8, s[16:17]
-; GFX8-NEXT: v_addc_u32_e64 v4, s[18:19], 0, 0, s[18:19]
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[18:19], s1, v17, 0
-; GFX8-NEXT: s_mul_i32 s1, s1, s14
-; GFX8-NEXT: v_addc_u32_e64 v8, s[16:17], v0, v5, s[16:17]
-; GFX8-NEXT: v_addc_u32_e64 v11, s[16:17], v1, v4, s[16:17]
-; GFX8-NEXT: v_add_u32_e64 v7, s[16:17], v7, v9
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[18:19], s3, v6, 0
-; GFX8-NEXT: v_addc_u32_e64 v6, s[16:17], 0, 0, s[16:17]
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[16:17], s2, v16, 0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, v4, v7, vcc
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, v5, v6, vcc
-; GFX8-NEXT: v_mad_u64_u32 v[6:7], s[16:17], s3, v16, v[1:2]
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v8
-; GFX8-NEXT: v_addc_u32_e32 v20, vcc, v5, v11, vcc
-; GFX8-NEXT: v_mov_b32_e32 v1, v6
-; GFX8-NEXT: v_mad_u64_u32 v[8:9], s[16:17], s2, v17, v[1:2]
-; GFX8-NEXT: v_add_u32_e64 v21, s[16:17], v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[18:19], s8, v4, 0
-; GFX8-NEXT: v_mov_b32_e32 v6, v2
-; GFX8-NEXT: v_addc_u32_e64 v8, s[20:21], v8, v20, s[16:17]
-; GFX8-NEXT: v_mad_u64_u32 v[11:12], s[18:19], s9, v4, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v1, v11
-; GFX8-NEXT: v_mov_b32_e32 v11, s0
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[18:19], s12, v11, 0
-; GFX8-NEXT: v_mad_u64_u32 v[5:6], s[18:19], s13, v11, v[5:6]
-; GFX8-NEXT: v_mov_b32_e32 v11, s5
-; GFX8-NEXT: v_mad_u64_u32 v[13:14], s[18:19], s8, v11, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
-; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[18:19], s12, v22, v[1:2]
-; GFX8-NEXT: v_add_u32_e64 v23, s[18:19], v4, v0
-; GFX8-NEXT: v_add_u32_e64 v4, s[16:17], v7, v9
-; GFX8-NEXT: v_addc_u32_e64 v5, s[16:17], 0, 0, s[16:17]
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[16:17], s3, v17, v[4:5]
-; GFX8-NEXT: s_and_b64 s[16:17], vcc, exec
-; GFX8-NEXT: s_cselect_b64 s[16:17], 1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s16
-; GFX8-NEXT: v_mov_b32_e32 v7, s17
-; GFX8-NEXT: v_addc_u32_e64 v9, vcc, v4, v0, s[20:21]
-; GFX8-NEXT: v_addc_u32_e32 v7, vcc, v5, v7, vcc
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s4, v16, 0
-; GFX8-NEXT: v_add_u32_e64 v0, s[20:21], v12, v14
-; GFX8-NEXT: v_addc_u32_e64 v1, s[18:19], v1, v13, s[18:19]
-; GFX8-NEXT: v_add_u32_e32 v12, vcc, v21, v23
-; GFX8-NEXT: v_addc_u32_e64 v8, s[16:17], v8, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s11, v5
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, s10, v1
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s6, v15, v[4:5]
-; GFX8-NEXT: v_addc_u32_e64 v1, s[4:5], 0, 0, s[20:21]
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s9, v11, v[0:1]
-; GFX8-NEXT: s_mul_i32 s6, s6, s9
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, s7, v5
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, s6, v5
-; GFX8-NEXT: v_add_u32_e32 v11, vcc, v0, v4
+; GFX8-NEXT: s_mov_b32 s16, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: v_mul_hi_u32 v2, s1, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s16, v1
+; GFX8-NEXT: v_readfirstlane_b32 s17, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s10
+; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX8-NEXT: v_readfirstlane_b32 s21, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mul_hi_u32 v3, v2, s8
+; GFX8-NEXT: s_mul_i32 s18, s16, s10
+; GFX8-NEXT: s_mul_i32 s20, s1, s9
+; GFX8-NEXT: v_readfirstlane_b32 s19, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: s_add_u32 s18, s20, s18
+; GFX8-NEXT: s_addc_u32 s19, s21, s19
+; GFX8-NEXT: s_mul_i32 s21, s2, s8
+; GFX8-NEXT: v_readfirstlane_b32 s23, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v0, s8
+; GFX8-NEXT: s_cselect_b32 s20, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s22, v3
+; GFX8-NEXT: s_add_u32 s18, s21, s18
+; GFX8-NEXT: s_addc_u32 s19, s22, s19
+; GFX8-NEXT: s_mul_i32 s22, s16, s9
+; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: s_add_u32 s17, s22, s17
+; GFX8-NEXT: s_addc_u32 s22, s23, s18
+; GFX8-NEXT: v_readfirstlane_b32 s23, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s12
+; GFX8-NEXT: v_mul_hi_u32 v3, s16, v1
+; GFX8-NEXT: s_mul_i32 s18, s1, s8
+; GFX8-NEXT: s_cselect_b32 s25, 1, 0
+; GFX8-NEXT: s_add_u32 s18, s18, s17
+; GFX8-NEXT: s_addc_u32 s17, s23, s22
+; GFX8-NEXT: v_mov_b32_e32 v4, s11
+; GFX8-NEXT: v_readfirstlane_b32 s23, v3
+; GFX8-NEXT: v_mul_hi_u32 v3, v2, s10
+; GFX8-NEXT: v_mul_hi_u32 v5, s1, v4
+; GFX8-NEXT: s_mul_i32 s22, s16, s12
+; GFX8-NEXT: s_mul_i32 s24, s1, s11
+; GFX8-NEXT: v_readfirstlane_b32 s28, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, s3
+; GFX8-NEXT: v_readfirstlane_b32 s27, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v3, s9
+; GFX8-NEXT: s_cselect_b32 s26, 1, 0
+; GFX8-NEXT: s_add_u32 s24, s24, s22
+; GFX8-NEXT: s_addc_u32 s23, s27, s23
+; GFX8-NEXT: v_readfirstlane_b32 s29, v5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: v_mul_hi_u32 v6, v5, s8
+; GFX8-NEXT: s_mul_i32 s27, s2, s10
+; GFX8-NEXT: s_cselect_b32 s22, 1, 0
+; GFX8-NEXT: s_add_u32 s24, s27, s24
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s10
+; GFX8-NEXT: s_addc_u32 s27, s28, s23
+; GFX8-NEXT: s_mul_i32 s28, s3, s9
+; GFX8-NEXT: s_cselect_b32 s23, 1, 0
+; GFX8-NEXT: s_add_u32 s28, s28, s24
+; GFX8-NEXT: v_readfirstlane_b32 s30, v6
+; GFX8-NEXT: v_mul_hi_u32 v6, s16, v4
+; GFX8-NEXT: s_addc_u32 s27, s29, s27
+; GFX8-NEXT: s_mul_i32 s29, s4, s8
+; GFX8-NEXT: s_cselect_b32 s24, 1, 0
+; GFX8-NEXT: s_add_u32 s28, s29, s28
+; GFX8-NEXT: v_readfirstlane_b32 s33, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, v2, s9
+; GFX8-NEXT: s_addc_u32 s27, s30, s27
+; GFX8-NEXT: s_mul_i32 s30, s16, s11
+; GFX8-NEXT: s_cselect_b32 s29, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s31, v6
+; GFX8-NEXT: s_add_u32 s19, s30, s19
+; GFX8-NEXT: s_addc_u32 s28, s31, s28
+; GFX8-NEXT: s_mul_i32 s31, s1, s10
+; GFX8-NEXT: s_cselect_b32 s30, 1, 0
+; GFX8-NEXT: s_add_u32 s19, s31, s19
+; GFX8-NEXT: v_readfirstlane_b32 s34, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, v3, s8
+; GFX8-NEXT: s_addc_u32 s28, s33, s28
+; GFX8-NEXT: s_mul_i32 s33, s2, s9
+; GFX8-NEXT: s_cselect_b32 s31, 1, 0
+; GFX8-NEXT: s_add_u32 s19, s33, s19
+; GFX8-NEXT: s_addc_u32 s28, s34, s28
+; GFX8-NEXT: s_mul_i32 s34, s3, s8
+; GFX8-NEXT: s_cselect_b32 s33, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s35, v0
+; GFX8-NEXT: s_add_u32 s19, s34, s19
+; GFX8-NEXT: s_addc_u32 s28, s35, s28
+; GFX8-NEXT: s_cselect_b32 s34, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s25, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s14
-; GFX8-NEXT: v_addc_u32_e32 v13, vcc, v1, v5, vcc
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s0, v0, 0
-; GFX8-NEXT: s_mul_i32 s0, s0, s15
-; GFX8-NEXT: v_mov_b32_e32 v4, s12
-; GFX8-NEXT: s_mul_i32 s3, s3, s12
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s1, v1
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v4, v[0:1]
-; GFX8-NEXT: v_add_u32_e64 v4, s[0:1], v6, v2
-; GFX8-NEXT: v_addc_u32_e64 v5, s[0:1], 0, 0, s[0:1]
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, s3, v1
-; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s13, v22, v[4:5]
+; GFX8-NEXT: s_cselect_b32 s25, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s26, 0
+; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX8-NEXT: s_addc_u32 s19, s25, s19
+; GFX8-NEXT: s_cselect_b32 s25, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s20, 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s13
+; GFX8-NEXT: s_cselect_b32 s20, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s21, 0
+; GFX8-NEXT: v_mul_hi_u32 v6, s1, v2
+; GFX8-NEXT: s_addc_u32 s20, s20, 0
+; GFX8-NEXT: v_readfirstlane_b32 s26, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v1
+; GFX8-NEXT: s_cmp_lg_u32 s25, 0
+; GFX8-NEXT: s_addc_u32 s20, s20, s28
+; GFX8-NEXT: s_mul_i32 s25, s16, s14
+; GFX8-NEXT: s_mul_i32 s28, s1, s13
+; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s35, v6
+; GFX8-NEXT: s_add_u32 s25, s28, s25
+; GFX8-NEXT: s_addc_u32 s26, s35, s26
+; GFX8-NEXT: v_readfirstlane_b32 s35, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, v3, s11
+; GFX8-NEXT: s_mul_i32 s28, s2, s12
+; GFX8-NEXT: s_add_u32 s25, s28, s25
+; GFX8-NEXT: s_addc_u32 s26, s35, s26
+; GFX8-NEXT: v_readfirstlane_b32 s35, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, v5, s10
+; GFX8-NEXT: s_mul_i32 s28, s3, s11
+; GFX8-NEXT: s_add_u32 s25, s28, s25
+; GFX8-NEXT: s_addc_u32 s26, s35, s26
+; GFX8-NEXT: v_readfirstlane_b32 s35, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
+; GFX8-NEXT: v_mul_hi_u32 v6, v0, s9
+; GFX8-NEXT: s_mul_i32 s28, s4, s10
+; GFX8-NEXT: s_add_u32 s25, s28, s25
+; GFX8-NEXT: v_mul_hi_u32 v1, s1, v1
+; GFX8-NEXT: s_addc_u32 s26, s35, s26
+; GFX8-NEXT: v_readfirstlane_b32 s35, v6
+; GFX8-NEXT: v_mov_b32_e32 v6, s6
+; GFX8-NEXT: v_mul_hi_u32 v6, v6, s8
+; GFX8-NEXT: s_mul_i32 s28, s5, s9
+; GFX8-NEXT: s_add_u32 s25, s28, s25
+; GFX8-NEXT: v_mul_hi_u32 v2, s16, v2
+; GFX8-NEXT: v_readfirstlane_b32 s36, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s2, v4
+; GFX8-NEXT: s_addc_u32 s26, s35, s26
+; GFX8-NEXT: s_mul_i32 s28, s6, s8
+; GFX8-NEXT: v_readfirstlane_b32 s35, v6
+; GFX8-NEXT: s_add_u32 s25, s28, s25
+; GFX8-NEXT: s_addc_u32 s26, s35, s26
+; GFX8-NEXT: s_mul_i32 s28, s16, s13
+; GFX8-NEXT: v_readfirstlane_b32 s35, v2
+; GFX8-NEXT: s_add_u32 s27, s28, s27
+; GFX8-NEXT: v_readfirstlane_b32 s37, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v3, s10
+; GFX8-NEXT: s_addc_u32 s25, s35, s25
+; GFX8-NEXT: s_mul_i32 s35, s1, s12
+; GFX8-NEXT: s_cselect_b32 s28, 1, 0
+; GFX8-NEXT: s_add_u32 s27, s35, s27
+; GFX8-NEXT: s_addc_u32 s25, s36, s25
+; GFX8-NEXT: s_mul_i32 s36, s2, s11
+; GFX8-NEXT: s_cselect_b32 s35, 1, 0
+; GFX8-NEXT: s_add_u32 s27, s36, s27
+; GFX8-NEXT: v_readfirstlane_b32 s38, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v5, s9
+; GFX8-NEXT: s_addc_u32 s25, s37, s25
+; GFX8-NEXT: s_mul_i32 s37, s3, s10
+; GFX8-NEXT: s_cselect_b32 s36, 1, 0
+; GFX8-NEXT: s_add_u32 s27, s37, s27
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s8
+; GFX8-NEXT: s_addc_u32 s25, s38, s25
+; GFX8-NEXT: s_mul_i32 s38, s4, s9
+; GFX8-NEXT: s_cselect_b32 s37, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s39, v1
+; GFX8-NEXT: s_add_u32 s27, s38, s27
+; GFX8-NEXT: s_addc_u32 s25, s39, s25
+; GFX8-NEXT: s_mul_i32 s39, s5, s8
+; GFX8-NEXT: s_cselect_b32 s38, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s40, v0
+; GFX8-NEXT: s_add_u32 s27, s39, s27
+; GFX8-NEXT: s_addc_u32 s25, s40, s25
+; GFX8-NEXT: s_cselect_b32 s39, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s30, 0
+; GFX8-NEXT: s_cselect_b32 s30, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s31, 0
+; GFX8-NEXT: s_addc_u32 s30, s30, 0
+; GFX8-NEXT: s_cmp_lg_u32 s33, 0
+; GFX8-NEXT: s_addc_u32 s30, s30, 0
+; GFX8-NEXT: s_cmp_lg_u32 s34, 0
+; GFX8-NEXT: s_addc_u32 s30, s30, 0
+; GFX8-NEXT: s_cmp_lg_u32 s21, 0
+; GFX8-NEXT: s_addc_u32 s21, s30, s27
+; GFX8-NEXT: s_cselect_b32 s27, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s22, 0
+; GFX8-NEXT: s_cselect_b32 s22, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s23, 0
+; GFX8-NEXT: s_addc_u32 s22, s22, 0
+; GFX8-NEXT: s_cmp_lg_u32 s24, 0
+; GFX8-NEXT: s_addc_u32 s22, s22, 0
+; GFX8-NEXT: s_cmp_lg_u32 s29, 0
+; GFX8-NEXT: s_addc_u32 s22, s22, 0
+; GFX8-NEXT: s_cmp_lg_u32 s27, 0
+; GFX8-NEXT: s_addc_u32 s22, s22, s25
+; GFX8-NEXT: s_mul_i32 s16, s16, s15
+; GFX8-NEXT: s_addc_u32 s15, s26, s16
+; GFX8-NEXT: s_mul_i32 s1, s1, s14
+; GFX8-NEXT: s_cmp_lg_u32 s39, 0
+; GFX8-NEXT: s_addc_u32 s1, s15, s1
; GFX8-NEXT: s_mul_i32 s2, s2, s13
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, s2, v6
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v2, v4, vcc
-; GFX8-NEXT: v_addc_u32_e64 v0, vcc, v0, v11, s[18:19]
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v13, vcc
-; GFX8-NEXT: v_addc_u32_e64 v0, vcc, v9, v0, s[16:17]
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GFX8-NEXT: v_readfirstlane_b32 s6, v0
-; GFX8-NEXT: v_readfirstlane_b32 s7, v1
-; GFX8-NEXT: v_readfirstlane_b32 s4, v12
-; GFX8-NEXT: v_readfirstlane_b32 s5, v8
-; GFX8-NEXT: v_readfirstlane_b32 s2, v18
-; GFX8-NEXT: v_readfirstlane_b32 s3, v19
-; GFX8-NEXT: v_readfirstlane_b32 s0, v3
-; GFX8-NEXT: v_readfirstlane_b32 s1, v10
+; GFX8-NEXT: s_cmp_lg_u32 s38, 0
+; GFX8-NEXT: s_addc_u32 s1, s1, s2
+; GFX8-NEXT: s_mul_i32 s3, s3, s12
+; GFX8-NEXT: s_cmp_lg_u32 s37, 0
+; GFX8-NEXT: s_addc_u32 s1, s1, s3
+; GFX8-NEXT: s_mul_i32 s4, s4, s11
+; GFX8-NEXT: s_cmp_lg_u32 s36, 0
+; GFX8-NEXT: s_addc_u32 s1, s1, s4
+; GFX8-NEXT: s_mul_i32 s5, s5, s10
+; GFX8-NEXT: s_cmp_lg_u32 s35, 0
+; GFX8-NEXT: s_addc_u32 s1, s1, s5
+; GFX8-NEXT: s_mul_i32 s6, s6, s9
+; GFX8-NEXT: s_cmp_lg_u32 s28, 0
+; GFX8-NEXT: s_addc_u32 s1, s1, s6
+; GFX8-NEXT: s_mul_i32 s7, s7, s8
+; GFX8-NEXT: s_mul_i32 s0, s0, s8
+; GFX8-NEXT: s_add_u32 s7, s7, s1
+; GFX8-NEXT: s_mov_b32 s1, s18
+; GFX8-NEXT: s_mov_b32 s2, s17
+; GFX8-NEXT: s_mov_b32 s3, s19
+; GFX8-NEXT: s_mov_b32 s4, s20
+; GFX8-NEXT: s_mov_b32 s5, s21
+; GFX8-NEXT: s_mov_b32 s6, s22
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_mul_i256:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mul_i32 s19, s3, s8
-; GFX9-NEXT: s_mul_hi_u32 s21, s2, s8
-; GFX9-NEXT: s_mul_hi_u32 s18, s3, s8
-; GFX9-NEXT: s_add_u32 s19, s19, s21
-; GFX9-NEXT: s_mul_i32 s17, s2, s9
-; GFX9-NEXT: s_addc_u32 s26, s18, 0
-; GFX9-NEXT: s_mul_hi_u32 s16, s2, s9
-; GFX9-NEXT: s_mov_b32 s20, 0
-; GFX9-NEXT: s_add_u32 s21, s17, s19
-; GFX9-NEXT: s_addc_u32 s27, s16, 0
-; GFX9-NEXT: s_mul_i32 s16, s2, s8
-; GFX9-NEXT: s_mov_b32 s17, s20
-; GFX9-NEXT: s_or_b64 s[18:19], s[16:17], s[20:21]
-; GFX9-NEXT: s_mul_i32 s22, s1, s8
-; GFX9-NEXT: s_mul_hi_u32 s23, s0, s8
-; GFX9-NEXT: s_mul_hi_u32 s21, s1, s8
-; GFX9-NEXT: s_add_u32 s22, s22, s23
-; GFX9-NEXT: s_mul_i32 s16, s0, s9
-; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: s_mul_hi_u32 s17, s0, s9
-; GFX9-NEXT: s_add_u32 s16, s16, s22
-; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: s_add_u32 s17, s21, s17
-; GFX9-NEXT: s_addc_u32 s21, 0, 0
-; GFX9-NEXT: s_mul_i32 s23, s1, s9
-; GFX9-NEXT: s_mul_hi_u32 s22, s1, s9
+; GFX9-NEXT: s_mov_b32 s16, s0
+; GFX9-NEXT: s_mul_i32 s18, s16, s10
+; GFX9-NEXT: s_mul_i32 s20, s1, s9
+; GFX9-NEXT: s_mul_hi_u32 s19, s16, s10
+; GFX9-NEXT: s_mul_hi_u32 s21, s1, s9
+; GFX9-NEXT: s_add_u32 s18, s20, s18
+; GFX9-NEXT: s_addc_u32 s19, s21, s19
+; GFX9-NEXT: s_mul_i32 s21, s2, s8
+; GFX9-NEXT: s_cselect_b32 s20, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s22, s2, s8
+; GFX9-NEXT: s_add_u32 s18, s21, s18
+; GFX9-NEXT: s_mul_hi_u32 s17, s16, s8
+; GFX9-NEXT: s_addc_u32 s19, s22, s19
+; GFX9-NEXT: s_mul_i32 s22, s16, s9
+; GFX9-NEXT: s_cselect_b32 s21, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s23, s16, s9
+; GFX9-NEXT: s_add_u32 s17, s22, s17
+; GFX9-NEXT: s_addc_u32 s18, s23, s18
+; GFX9-NEXT: s_mul_i32 s23, s1, s8
+; GFX9-NEXT: s_cselect_b32 s22, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s24, s1, s8
; GFX9-NEXT: s_add_u32 s17, s23, s17
-; GFX9-NEXT: s_addc_u32 s21, s22, s21
-; GFX9-NEXT: s_add_u32 s17, s18, s17
-; GFX9-NEXT: s_addc_u32 s24, s19, s21
-; GFX9-NEXT: s_mul_i32 s25, s1, s10
-; GFX9-NEXT: s_mul_hi_u32 s28, s0, s10
-; GFX9-NEXT: s_cselect_b64 s[22:23], -1, 0
-; GFX9-NEXT: s_mul_hi_u32 s21, s1, s10
-; GFX9-NEXT: s_add_u32 s25, s25, s28
-; GFX9-NEXT: s_mul_i32 s19, s0, s11
-; GFX9-NEXT: s_addc_u32 s28, s21, 0
-; GFX9-NEXT: s_mul_hi_u32 s18, s0, s11
-; GFX9-NEXT: s_add_u32 s21, s19, s25
-; GFX9-NEXT: s_addc_u32 s29, s18, 0
-; GFX9-NEXT: s_mul_i32 s18, s0, s10
-; GFX9-NEXT: s_mov_b32 s19, s20
-; GFX9-NEXT: s_or_b64 s[18:19], s[18:19], s[20:21]
-; GFX9-NEXT: s_add_u32 s17, s18, s17
-; GFX9-NEXT: s_addc_u32 s18, s19, s24
-; GFX9-NEXT: s_cselect_b64 s[24:25], -1, 0
-; GFX9-NEXT: s_add_u32 s19, s28, s29
-; GFX9-NEXT: s_addc_u32 s21, 0, 0
-; GFX9-NEXT: s_mul_i32 s29, s1, s11
-; GFX9-NEXT: s_cmp_lg_u64 s[24:25], 0
-; GFX9-NEXT: s_mul_hi_u32 s28, s1, s11
-; GFX9-NEXT: s_addc_u32 s19, s29, s19
-; GFX9-NEXT: s_addc_u32 s21, s28, s21
-; GFX9-NEXT: s_add_u32 s24, s26, s27
-; GFX9-NEXT: s_addc_u32 s25, 0, 0
-; GFX9-NEXT: s_mul_i32 s27, s3, s9
-; GFX9-NEXT: s_cmp_lg_u64 s[22:23], 0
-; GFX9-NEXT: s_mul_hi_u32 s26, s3, s9
-; GFX9-NEXT: s_addc_u32 s22, s27, s24
-; GFX9-NEXT: s_addc_u32 s23, s26, s25
-; GFX9-NEXT: s_add_u32 s19, s22, s19
-; GFX9-NEXT: s_addc_u32 s26, s23, s21
-; GFX9-NEXT: s_mul_i32 s27, s3, s10
+; GFX9-NEXT: s_addc_u32 s18, s24, s18
+; GFX9-NEXT: s_mul_i32 s24, s16, s12
+; GFX9-NEXT: s_mul_i32 s26, s1, s11
+; GFX9-NEXT: s_cselect_b32 s23, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s25, s16, s12
+; GFX9-NEXT: s_mul_hi_u32 s27, s1, s11
+; GFX9-NEXT: s_add_u32 s24, s26, s24
+; GFX9-NEXT: s_addc_u32 s25, s27, s25
+; GFX9-NEXT: s_mul_i32 s27, s2, s10
+; GFX9-NEXT: s_cselect_b32 s26, 1, 0
; GFX9-NEXT: s_mul_hi_u32 s28, s2, s10
-; GFX9-NEXT: s_cselect_b64 s[22:23], -1, 0
-; GFX9-NEXT: s_mul_hi_u32 s25, s3, s10
-; GFX9-NEXT: s_add_u32 s27, s27, s28
-; GFX9-NEXT: s_mul_i32 s21, s2, s11
-; GFX9-NEXT: s_addc_u32 s30, s25, 0
-; GFX9-NEXT: s_mul_hi_u32 s24, s2, s11
-; GFX9-NEXT: s_add_u32 s21, s21, s27
-; GFX9-NEXT: s_addc_u32 s31, s24, 0
-; GFX9-NEXT: s_mul_i32 s24, s2, s10
-; GFX9-NEXT: s_mov_b32 s25, s20
-; GFX9-NEXT: s_or_b64 s[24:25], s[24:25], s[20:21]
-; GFX9-NEXT: s_add_u32 s19, s24, s19
-; GFX9-NEXT: s_addc_u32 s33, s25, s26
-; GFX9-NEXT: s_mul_i32 s28, s9, s4
-; GFX9-NEXT: s_mul_hi_u32 s29, s8, s4
-; GFX9-NEXT: s_cselect_b64 s[24:25], -1, 0
-; GFX9-NEXT: s_mul_hi_u32 s27, s9, s4
-; GFX9-NEXT: s_add_u32 s28, s28, s29
-; GFX9-NEXT: s_mul_i32 s21, s8, s5
-; GFX9-NEXT: s_addc_u32 s34, s27, 0
-; GFX9-NEXT: s_mul_hi_u32 s26, s8, s5
-; GFX9-NEXT: s_add_u32 s21, s21, s28
-; GFX9-NEXT: s_addc_u32 s35, s26, 0
-; GFX9-NEXT: s_mul_i32 s26, s8, s4
-; GFX9-NEXT: s_mov_b32 s27, s20
-; GFX9-NEXT: s_or_b64 s[26:27], s[26:27], s[20:21]
-; GFX9-NEXT: s_mul_i32 s36, s13, s0
-; GFX9-NEXT: s_mul_hi_u32 s37, s12, s0
-; GFX9-NEXT: s_mul_hi_u32 s29, s13, s0
-; GFX9-NEXT: s_add_u32 s36, s36, s37
-; GFX9-NEXT: s_mul_i32 s21, s12, s1
-; GFX9-NEXT: s_addc_u32 s37, s29, 0
-; GFX9-NEXT: s_mul_hi_u32 s28, s12, s1
-; GFX9-NEXT: s_add_u32 s21, s21, s36
-; GFX9-NEXT: s_addc_u32 s36, s28, 0
-; GFX9-NEXT: s_mul_i32 s28, s12, s0
-; GFX9-NEXT: s_mov_b32 s29, s20
-; GFX9-NEXT: s_or_b64 s[20:21], s[28:29], s[20:21]
-; GFX9-NEXT: s_add_u32 s20, s20, s26
-; GFX9-NEXT: s_addc_u32 s21, s21, s27
-; GFX9-NEXT: s_cselect_b64 s[26:27], -1, 0
-; GFX9-NEXT: s_add_u32 s19, s19, s20
-; GFX9-NEXT: s_addc_u32 s20, s33, s21
-; GFX9-NEXT: s_cselect_b64 s[28:29], -1, 0
-; GFX9-NEXT: s_and_b64 s[22:23], s[22:23], exec
-; GFX9-NEXT: s_cselect_b64 s[22:23], 1, 0
-; GFX9-NEXT: s_add_u32 s21, s30, s31
-; GFX9-NEXT: s_addc_u32 s30, 0, 0
-; GFX9-NEXT: s_mul_i32 s33, s3, s11
-; GFX9-NEXT: s_mul_hi_u32 s31, s3, s11
-; GFX9-NEXT: s_add_u32 s21, s33, s21
-; GFX9-NEXT: s_addc_u32 s30, s31, s30
-; GFX9-NEXT: s_cmp_lg_u64 s[24:25], 0
-; GFX9-NEXT: s_addc_u32 s21, s21, s22
-; GFX9-NEXT: s_addc_u32 s22, s30, s23
-; GFX9-NEXT: s_mul_i32 s11, s4, s11
-; GFX9-NEXT: s_mul_hi_u32 s23, s4, s10
-; GFX9-NEXT: s_add_i32 s11, s23, s11
-; GFX9-NEXT: s_mul_i32 s23, s5, s10
-; GFX9-NEXT: s_add_i32 s11, s11, s23
-; GFX9-NEXT: s_mul_i32 s4, s4, s10
-; GFX9-NEXT: s_mul_i32 s10, s6, s9
-; GFX9-NEXT: s_mul_hi_u32 s23, s6, s8
-; GFX9-NEXT: s_add_i32 s10, s23, s10
-; GFX9-NEXT: s_mul_i32 s7, s7, s8
-; GFX9-NEXT: s_add_i32 s10, s10, s7
-; GFX9-NEXT: s_mul_i32 s6, s6, s8
-; GFX9-NEXT: s_add_u32 s4, s6, s4
-; GFX9-NEXT: s_addc_u32 s6, s10, s11
-; GFX9-NEXT: s_add_u32 s7, s34, s35
-; GFX9-NEXT: s_addc_u32 s10, 0, 0
-; GFX9-NEXT: s_mul_hi_u32 s11, s9, s5
-; GFX9-NEXT: s_mul_i32 s9, s9, s5
-; GFX9-NEXT: s_add_u32 s5, s9, s7
-; GFX9-NEXT: s_addc_u32 s7, s11, s10
-; GFX9-NEXT: s_add_u32 s4, s5, s4
-; GFX9-NEXT: s_addc_u32 s5, s7, s6
-; GFX9-NEXT: s_mul_i32 s6, s0, s15
-; GFX9-NEXT: s_mul_hi_u32 s7, s0, s14
-; GFX9-NEXT: s_mul_i32 s9, s2, s13
-; GFX9-NEXT: s_mul_hi_u32 s10, s2, s12
-; GFX9-NEXT: s_add_i32 s6, s7, s6
-; GFX9-NEXT: s_mul_i32 s7, s1, s14
-; GFX9-NEXT: s_add_i32 s9, s10, s9
+; GFX9-NEXT: s_add_u32 s24, s27, s24
+; GFX9-NEXT: s_addc_u32 s25, s28, s25
+; GFX9-NEXT: s_mul_i32 s28, s3, s9
+; GFX9-NEXT: s_cselect_b32 s27, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s29, s3, s9
+; GFX9-NEXT: s_add_u32 s24, s28, s24
+; GFX9-NEXT: s_addc_u32 s25, s29, s25
+; GFX9-NEXT: s_mul_i32 s29, s4, s8
+; GFX9-NEXT: s_cselect_b32 s28, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s30, s4, s8
+; GFX9-NEXT: s_add_u32 s24, s29, s24
+; GFX9-NEXT: s_addc_u32 s25, s30, s25
+; GFX9-NEXT: s_mul_i32 s30, s16, s11
+; GFX9-NEXT: s_cselect_b32 s29, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s31, s16, s11
+; GFX9-NEXT: s_add_u32 s19, s30, s19
+; GFX9-NEXT: s_addc_u32 s24, s31, s24
+; GFX9-NEXT: s_mul_i32 s31, s1, s10
+; GFX9-NEXT: s_cselect_b32 s30, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s33, s1, s10
+; GFX9-NEXT: s_add_u32 s19, s31, s19
+; GFX9-NEXT: s_addc_u32 s24, s33, s24
+; GFX9-NEXT: s_mul_i32 s33, s2, s9
+; GFX9-NEXT: s_cselect_b32 s31, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s34, s2, s9
+; GFX9-NEXT: s_add_u32 s19, s33, s19
+; GFX9-NEXT: s_addc_u32 s24, s34, s24
+; GFX9-NEXT: s_mul_i32 s34, s3, s8
+; GFX9-NEXT: s_cselect_b32 s33, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s35, s3, s8
+; GFX9-NEXT: s_add_u32 s19, s34, s19
+; GFX9-NEXT: s_addc_u32 s24, s35, s24
+; GFX9-NEXT: s_cselect_b32 s34, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s22, 0
+; GFX9-NEXT: s_cselect_b32 s22, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s23, 0
+; GFX9-NEXT: s_addc_u32 s19, s22, s19
+; GFX9-NEXT: s_cselect_b32 s22, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s20, 0
+; GFX9-NEXT: s_cselect_b32 s20, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s21, 0
+; GFX9-NEXT: s_addc_u32 s20, s20, 0
+; GFX9-NEXT: s_cmp_lg_u32 s22, 0
+; GFX9-NEXT: s_addc_u32 s20, s20, s24
+; GFX9-NEXT: s_mul_i32 s22, s16, s14
+; GFX9-NEXT: s_mul_i32 s24, s1, s13
+; GFX9-NEXT: s_cselect_b32 s21, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s23, s16, s14
+; GFX9-NEXT: s_mul_hi_u32 s35, s1, s13
+; GFX9-NEXT: s_add_u32 s22, s24, s22
+; GFX9-NEXT: s_addc_u32 s23, s35, s23
+; GFX9-NEXT: s_mul_i32 s24, s2, s12
+; GFX9-NEXT: s_mul_hi_u32 s35, s2, s12
+; GFX9-NEXT: s_add_u32 s22, s24, s22
+; GFX9-NEXT: s_addc_u32 s23, s35, s23
+; GFX9-NEXT: s_mul_i32 s24, s3, s11
+; GFX9-NEXT: s_mul_hi_u32 s35, s3, s11
+; GFX9-NEXT: s_add_u32 s22, s24, s22
+; GFX9-NEXT: s_addc_u32 s23, s35, s23
+; GFX9-NEXT: s_mul_i32 s24, s4, s10
+; GFX9-NEXT: s_mul_hi_u32 s35, s4, s10
+; GFX9-NEXT: s_add_u32 s22, s24, s22
+; GFX9-NEXT: s_addc_u32 s23, s35, s23
+; GFX9-NEXT: s_mul_i32 s24, s5, s9
+; GFX9-NEXT: s_mul_hi_u32 s35, s5, s9
+; GFX9-NEXT: s_add_u32 s22, s24, s22
+; GFX9-NEXT: s_addc_u32 s23, s35, s23
+; GFX9-NEXT: s_mul_i32 s24, s6, s8
+; GFX9-NEXT: s_mul_hi_u32 s35, s6, s8
+; GFX9-NEXT: s_add_u32 s22, s24, s22
+; GFX9-NEXT: s_addc_u32 s23, s35, s23
+; GFX9-NEXT: s_mul_i32 s24, s16, s13
+; GFX9-NEXT: s_mul_hi_u32 s35, s16, s13
+; GFX9-NEXT: s_add_u32 s24, s24, s25
+; GFX9-NEXT: s_addc_u32 s22, s35, s22
+; GFX9-NEXT: s_mul_i32 s35, s1, s12
+; GFX9-NEXT: s_cselect_b32 s25, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s36, s1, s12
+; GFX9-NEXT: s_add_u32 s24, s35, s24
+; GFX9-NEXT: s_addc_u32 s22, s36, s22
+; GFX9-NEXT: s_mul_i32 s36, s2, s11
+; GFX9-NEXT: s_cselect_b32 s35, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s37, s2, s11
+; GFX9-NEXT: s_add_u32 s24, s36, s24
+; GFX9-NEXT: s_addc_u32 s22, s37, s22
+; GFX9-NEXT: s_mul_i32 s37, s3, s10
+; GFX9-NEXT: s_cselect_b32 s36, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s38, s3, s10
+; GFX9-NEXT: s_add_u32 s24, s37, s24
+; GFX9-NEXT: s_addc_u32 s22, s38, s22
+; GFX9-NEXT: s_mul_i32 s38, s4, s9
+; GFX9-NEXT: s_cselect_b32 s37, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s39, s4, s9
+; GFX9-NEXT: s_add_u32 s24, s38, s24
+; GFX9-NEXT: s_addc_u32 s22, s39, s22
+; GFX9-NEXT: s_mul_i32 s39, s5, s8
+; GFX9-NEXT: s_cselect_b32 s38, 1, 0
+; GFX9-NEXT: s_mul_hi_u32 s40, s5, s8
+; GFX9-NEXT: s_add_u32 s24, s39, s24
+; GFX9-NEXT: s_addc_u32 s22, s40, s22
+; GFX9-NEXT: s_cselect_b32 s39, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s30, 0
+; GFX9-NEXT: s_cselect_b32 s30, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s31, 0
+; GFX9-NEXT: s_addc_u32 s30, s30, 0
+; GFX9-NEXT: s_cmp_lg_u32 s33, 0
+; GFX9-NEXT: s_addc_u32 s30, s30, 0
+; GFX9-NEXT: s_cmp_lg_u32 s34, 0
+; GFX9-NEXT: s_addc_u32 s30, s30, 0
+; GFX9-NEXT: s_cmp_lg_u32 s21, 0
+; GFX9-NEXT: s_addc_u32 s21, s30, s24
+; GFX9-NEXT: s_cselect_b32 s24, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s26, 0
+; GFX9-NEXT: s_cselect_b32 s26, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s27, 0
+; GFX9-NEXT: s_addc_u32 s26, s26, 0
+; GFX9-NEXT: s_cmp_lg_u32 s28, 0
+; GFX9-NEXT: s_addc_u32 s26, s26, 0
+; GFX9-NEXT: s_cmp_lg_u32 s29, 0
+; GFX9-NEXT: s_addc_u32 s26, s26, 0
+; GFX9-NEXT: s_cmp_lg_u32 s24, 0
+; GFX9-NEXT: s_addc_u32 s22, s26, s22
+; GFX9-NEXT: s_mul_i32 s16, s16, s15
+; GFX9-NEXT: s_addc_u32 s15, s23, s16
+; GFX9-NEXT: s_mul_i32 s1, s1, s14
+; GFX9-NEXT: s_cmp_lg_u32 s39, 0
+; GFX9-NEXT: s_addc_u32 s1, s15, s1
+; GFX9-NEXT: s_mul_i32 s2, s2, s13
+; GFX9-NEXT: s_cmp_lg_u32 s38, 0
+; GFX9-NEXT: s_addc_u32 s1, s1, s2
; GFX9-NEXT: s_mul_i32 s3, s3, s12
-; GFX9-NEXT: s_add_i32 s6, s6, s7
-; GFX9-NEXT: s_mul_i32 s7, s0, s14
-; GFX9-NEXT: s_add_i32 s9, s9, s3
-; GFX9-NEXT: s_mul_i32 s2, s2, s12
-; GFX9-NEXT: s_add_u32 s2, s2, s7
-; GFX9-NEXT: s_addc_u32 s3, s9, s6
-; GFX9-NEXT: s_add_u32 s6, s37, s36
-; GFX9-NEXT: s_addc_u32 s7, 0, 0
-; GFX9-NEXT: s_mul_hi_u32 s9, s13, s1
-; GFX9-NEXT: s_mul_i32 s13, s13, s1
-; GFX9-NEXT: s_add_u32 s1, s13, s6
-; GFX9-NEXT: s_addc_u32 s6, s9, s7
-; GFX9-NEXT: s_add_u32 s1, s1, s2
-; GFX9-NEXT: s_addc_u32 s2, s6, s3
-; GFX9-NEXT: s_cmp_lg_u64 s[26:27], 0
+; GFX9-NEXT: s_cmp_lg_u32 s37, 0
+; GFX9-NEXT: s_addc_u32 s1, s1, s3
+; GFX9-NEXT: s_mul_i32 s4, s4, s11
+; GFX9-NEXT: s_cmp_lg_u32 s36, 0
; GFX9-NEXT: s_addc_u32 s1, s1, s4
-; GFX9-NEXT: s_addc_u32 s2, s2, s5
-; GFX9-NEXT: s_cmp_lg_u64 s[28:29], 0
-; GFX9-NEXT: s_addc_u32 s6, s21, s1
-; GFX9-NEXT: s_addc_u32 s7, s22, s2
+; GFX9-NEXT: s_mul_i32 s5, s5, s10
+; GFX9-NEXT: s_cmp_lg_u32 s35, 0
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: s_mul_i32 s6, s6, s9
+; GFX9-NEXT: s_cmp_lg_u32 s25, 0
+; GFX9-NEXT: s_addc_u32 s1, s1, s6
+; GFX9-NEXT: s_mul_i32 s7, s7, s8
; GFX9-NEXT: s_mul_i32 s0, s0, s8
-; GFX9-NEXT: s_mov_b32 s1, s16
-; GFX9-NEXT: s_mov_b32 s2, s17
-; GFX9-NEXT: s_mov_b32 s3, s18
-; GFX9-NEXT: s_mov_b32 s4, s19
-; GFX9-NEXT: s_mov_b32 s5, s20
+; GFX9-NEXT: s_add_u32 s7, s7, s1
+; GFX9-NEXT: s_mov_b32 s1, s17
+; GFX9-NEXT: s_mov_b32 s2, s18
+; GFX9-NEXT: s_mov_b32 s3, s19
+; GFX9-NEXT: s_mov_b32 s4, s20
+; GFX9-NEXT: s_mov_b32 s5, s21
+; GFX9-NEXT: s_mov_b32 s6, s22
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_mul_i256:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_mul_i32 s19, s3, s8
+; GFX10PLUS-NEXT: s_mul_i32 s17, s0, s10
+; GFX10PLUS-NEXT: s_mul_i32 s19, s1, s9
+; GFX10PLUS-NEXT: s_mul_hi_u32 s18, s0, s10
+; GFX10PLUS-NEXT: s_mul_hi_u32 s20, s1, s9
+; GFX10PLUS-NEXT: s_add_u32 s17, s19, s17
+; GFX10PLUS-NEXT: s_addc_u32 s18, s20, s18
+; GFX10PLUS-NEXT: s_mul_i32 s20, s2, s8
; GFX10PLUS-NEXT: s_mul_hi_u32 s21, s2, s8
-; GFX10PLUS-NEXT: s_mul_hi_u32 s17, s3, s8
-; GFX10PLUS-NEXT: s_mul_i32 s16, s2, s9
-; GFX10PLUS-NEXT: s_mov_b32 s20, 0
-; GFX10PLUS-NEXT: s_add_u32 s19, s19, s21
-; GFX10PLUS-NEXT: s_addc_u32 s22, s17, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s18, s2, s9
-; GFX10PLUS-NEXT: s_add_u32 s21, s16, s19
-; GFX10PLUS-NEXT: s_mul_i32 s16, s2, s8
-; GFX10PLUS-NEXT: s_mov_b32 s17, s20
-; GFX10PLUS-NEXT: s_addc_u32 s23, s18, 0
-; GFX10PLUS-NEXT: s_or_b64 s[18:19], s[16:17], s[20:21]
-; GFX10PLUS-NEXT: s_mul_i32 s21, s1, s8
-; GFX10PLUS-NEXT: s_mul_hi_u32 s24, s0, s8
-; GFX10PLUS-NEXT: s_mul_hi_u32 s25, s1, s8
-; GFX10PLUS-NEXT: s_mul_i32 s16, s0, s9
-; GFX10PLUS-NEXT: s_add_u32 s21, s21, s24
-; GFX10PLUS-NEXT: s_mul_hi_u32 s17, s0, s9
-; GFX10PLUS-NEXT: s_addc_u32 s24, s25, 0
-; GFX10PLUS-NEXT: s_add_u32 s16, s16, s21
-; GFX10PLUS-NEXT: s_addc_u32 s17, s17, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s25, s1, s9
-; GFX10PLUS-NEXT: s_add_u32 s17, s24, s17
-; GFX10PLUS-NEXT: s_mul_i32 s24, s1, s9
-; GFX10PLUS-NEXT: s_addc_u32 s21, 0, 0
-; GFX10PLUS-NEXT: s_add_u32 s17, s24, s17
-; GFX10PLUS-NEXT: s_addc_u32 s21, s25, s21
-; GFX10PLUS-NEXT: s_add_u32 s17, s18, s17
-; GFX10PLUS-NEXT: s_addc_u32 s24, s19, s21
-; GFX10PLUS-NEXT: s_mul_i32 s19, s1, s10
-; GFX10PLUS-NEXT: s_mul_hi_u32 s21, s0, s10
-; GFX10PLUS-NEXT: s_mul_hi_u32 s27, s1, s10
-; GFX10PLUS-NEXT: s_cselect_b32 s25, -1, 0
-; GFX10PLUS-NEXT: s_mul_i32 s18, s0, s11
-; GFX10PLUS-NEXT: s_add_u32 s19, s19, s21
-; GFX10PLUS-NEXT: s_addc_u32 s27, s27, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s26, s0, s11
-; GFX10PLUS-NEXT: s_add_u32 s21, s18, s19
-; GFX10PLUS-NEXT: s_mul_i32 s18, s0, s10
-; GFX10PLUS-NEXT: s_mov_b32 s19, s20
-; GFX10PLUS-NEXT: s_addc_u32 s26, s26, 0
-; GFX10PLUS-NEXT: s_or_b64 s[18:19], s[18:19], s[20:21]
-; GFX10PLUS-NEXT: s_mul_hi_u32 s35, s13, s0
-; GFX10PLUS-NEXT: s_add_u32 s17, s18, s17
-; GFX10PLUS-NEXT: s_addc_u32 s18, s19, s24
-; GFX10PLUS-NEXT: s_cselect_b32 s19, -1, 0
-; GFX10PLUS-NEXT: s_add_u32 s21, s27, s26
-; GFX10PLUS-NEXT: s_addc_u32 s24, 0, 0
-; GFX10PLUS-NEXT: s_mul_i32 s26, s1, s11
+; GFX10PLUS-NEXT: s_cselect_b32 s19, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s17, s20, s17
+; GFX10PLUS-NEXT: s_mul_hi_u32 s16, s0, s8
+; GFX10PLUS-NEXT: s_addc_u32 s18, s21, s18
+; GFX10PLUS-NEXT: s_mul_i32 s21, s0, s9
+; GFX10PLUS-NEXT: s_mul_hi_u32 s22, s0, s9
+; GFX10PLUS-NEXT: s_cselect_b32 s20, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s16, s21, s16
+; GFX10PLUS-NEXT: s_addc_u32 s17, s22, s17
+; GFX10PLUS-NEXT: s_mul_i32 s22, s1, s8
+; GFX10PLUS-NEXT: s_mul_hi_u32 s23, s1, s8
+; GFX10PLUS-NEXT: s_cselect_b32 s21, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s16, s22, s16
+; GFX10PLUS-NEXT: s_addc_u32 s17, s23, s17
+; GFX10PLUS-NEXT: s_mul_i32 s23, s0, s12
+; GFX10PLUS-NEXT: s_mul_i32 s25, s1, s11
+; GFX10PLUS-NEXT: s_mul_hi_u32 s24, s0, s12
+; GFX10PLUS-NEXT: s_mul_hi_u32 s26, s1, s11
+; GFX10PLUS-NEXT: s_cselect_b32 s22, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s25, s23
+; GFX10PLUS-NEXT: s_addc_u32 s24, s26, s24
+; GFX10PLUS-NEXT: s_mul_i32 s26, s2, s10
+; GFX10PLUS-NEXT: s_mul_hi_u32 s27, s2, s10
+; GFX10PLUS-NEXT: s_cselect_b32 s25, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s26, s23
+; GFX10PLUS-NEXT: s_addc_u32 s24, s27, s24
+; GFX10PLUS-NEXT: s_mul_i32 s27, s3, s9
+; GFX10PLUS-NEXT: s_mul_hi_u32 s28, s3, s9
+; GFX10PLUS-NEXT: s_cselect_b32 s26, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s27, s23
+; GFX10PLUS-NEXT: s_addc_u32 s24, s28, s24
+; GFX10PLUS-NEXT: s_mul_i32 s28, s4, s8
+; GFX10PLUS-NEXT: s_mul_hi_u32 s29, s4, s8
+; GFX10PLUS-NEXT: s_cselect_b32 s27, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s28, s23
+; GFX10PLUS-NEXT: s_addc_u32 s24, s29, s24
+; GFX10PLUS-NEXT: s_mul_i32 s29, s0, s11
+; GFX10PLUS-NEXT: s_mul_hi_u32 s30, s0, s11
+; GFX10PLUS-NEXT: s_cselect_b32 s28, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s18, s29, s18
+; GFX10PLUS-NEXT: s_addc_u32 s23, s30, s23
+; GFX10PLUS-NEXT: s_mul_i32 s30, s1, s10
+; GFX10PLUS-NEXT: s_mul_hi_u32 s31, s1, s10
+; GFX10PLUS-NEXT: s_cselect_b32 s29, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s18, s30, s18
+; GFX10PLUS-NEXT: s_addc_u32 s23, s31, s23
+; GFX10PLUS-NEXT: s_mul_i32 s31, s2, s9
+; GFX10PLUS-NEXT: s_mul_hi_u32 s33, s2, s9
+; GFX10PLUS-NEXT: s_cselect_b32 s30, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s18, s31, s18
+; GFX10PLUS-NEXT: s_addc_u32 s23, s33, s23
+; GFX10PLUS-NEXT: s_mul_i32 s33, s3, s8
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s3, s8
+; GFX10PLUS-NEXT: s_cselect_b32 s31, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s18, s33, s18
+; GFX10PLUS-NEXT: s_addc_u32 s23, s34, s23
+; GFX10PLUS-NEXT: s_cselect_b32 s33, 1, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s21, 0
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s1, s13
+; GFX10PLUS-NEXT: s_cselect_b32 s21, 1, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s22, 0
+; GFX10PLUS-NEXT: s_mul_hi_u32 s22, s0, s14
+; GFX10PLUS-NEXT: s_addc_u32 s18, s21, s18
+; GFX10PLUS-NEXT: s_cselect_b32 s21, 1, 0
; GFX10PLUS-NEXT: s_cmp_lg_u32 s19, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s19, s1, s11
-; GFX10PLUS-NEXT: s_addc_u32 s21, s26, s21
-; GFX10PLUS-NEXT: s_addc_u32 s19, s19, s24
-; GFX10PLUS-NEXT: s_add_u32 s22, s22, s23
-; GFX10PLUS-NEXT: s_addc_u32 s23, 0, 0
-; GFX10PLUS-NEXT: s_mul_i32 s24, s3, s9
-; GFX10PLUS-NEXT: s_cmp_lg_u32 s25, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s25, s3, s9
-; GFX10PLUS-NEXT: s_addc_u32 s22, s24, s22
-; GFX10PLUS-NEXT: s_addc_u32 s23, s25, s23
-; GFX10PLUS-NEXT: s_add_u32 s24, s22, s21
-; GFX10PLUS-NEXT: s_addc_u32 s19, s23, s19
-; GFX10PLUS-NEXT: s_mul_i32 s22, s3, s10
-; GFX10PLUS-NEXT: s_mul_hi_u32 s23, s2, s10
-; GFX10PLUS-NEXT: s_mul_hi_u32 s27, s3, s10
-; GFX10PLUS-NEXT: s_cselect_b32 s26, -1, 0
-; GFX10PLUS-NEXT: s_mul_i32 s21, s2, s11
-; GFX10PLUS-NEXT: s_add_u32 s22, s22, s23
-; GFX10PLUS-NEXT: s_addc_u32 s27, s27, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s25, s2, s11
-; GFX10PLUS-NEXT: s_add_u32 s21, s21, s22
-; GFX10PLUS-NEXT: s_mul_i32 s22, s2, s10
-; GFX10PLUS-NEXT: s_mov_b32 s23, s20
-; GFX10PLUS-NEXT: s_addc_u32 s28, s25, 0
-; GFX10PLUS-NEXT: s_or_b64 s[22:23], s[22:23], s[20:21]
-; GFX10PLUS-NEXT: s_mul_i32 s21, s8, s5
-; GFX10PLUS-NEXT: s_add_u32 s29, s22, s24
-; GFX10PLUS-NEXT: s_addc_u32 s30, s23, s19
-; GFX10PLUS-NEXT: s_mul_i32 s22, s9, s4
-; GFX10PLUS-NEXT: s_mul_hi_u32 s23, s8, s4
-; GFX10PLUS-NEXT: s_mul_hi_u32 s24, s9, s4
-; GFX10PLUS-NEXT: s_cselect_b32 s31, -1, 0
-; GFX10PLUS-NEXT: s_add_u32 s22, s22, s23
-; GFX10PLUS-NEXT: s_addc_u32 s33, s24, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s19, s8, s5
-; GFX10PLUS-NEXT: s_add_u32 s21, s21, s22
-; GFX10PLUS-NEXT: s_mul_i32 s22, s8, s4
-; GFX10PLUS-NEXT: s_mov_b32 s23, s20
-; GFX10PLUS-NEXT: s_mul_i32 s24, s13, s0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s25, s12, s0
-; GFX10PLUS-NEXT: s_addc_u32 s34, s19, 0
-; GFX10PLUS-NEXT: s_or_b64 s[22:23], s[22:23], s[20:21]
-; GFX10PLUS-NEXT: s_mul_i32 s21, s12, s1
-; GFX10PLUS-NEXT: s_add_u32 s24, s24, s25
-; GFX10PLUS-NEXT: s_addc_u32 s35, s35, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s19, s12, s1
-; GFX10PLUS-NEXT: s_add_u32 s21, s21, s24
-; GFX10PLUS-NEXT: s_mul_i32 s24, s12, s0
-; GFX10PLUS-NEXT: s_mov_b32 s25, s20
-; GFX10PLUS-NEXT: s_addc_u32 s36, s19, 0
-; GFX10PLUS-NEXT: s_or_b64 s[20:21], s[24:25], s[20:21]
-; GFX10PLUS-NEXT: s_mul_i32 s7, s7, s8
-; GFX10PLUS-NEXT: s_add_u32 s19, s20, s22
-; GFX10PLUS-NEXT: s_addc_u32 s20, s21, s23
-; GFX10PLUS-NEXT: s_cselect_b32 s21, -1, 0
-; GFX10PLUS-NEXT: s_add_u32 s19, s29, s19
-; GFX10PLUS-NEXT: s_addc_u32 s20, s30, s20
-; GFX10PLUS-NEXT: s_cselect_b32 s24, -1, 0
-; GFX10PLUS-NEXT: s_and_b32 s22, s26, exec_lo
-; GFX10PLUS-NEXT: s_cselect_b64 s[22:23], 1, 0
-; GFX10PLUS-NEXT: s_add_u32 s25, s27, s28
-; GFX10PLUS-NEXT: s_mul_i32 s27, s3, s11
-; GFX10PLUS-NEXT: s_addc_u32 s26, 0, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s28, s3, s11
-; GFX10PLUS-NEXT: s_add_u32 s25, s27, s25
-; GFX10PLUS-NEXT: s_addc_u32 s26, s28, s26
+; GFX10PLUS-NEXT: s_mul_hi_u32 s35, s1, s12
+; GFX10PLUS-NEXT: s_cselect_b32 s19, 1, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s20, 0
+; GFX10PLUS-NEXT: s_mul_hi_u32 s36, s2, s11
+; GFX10PLUS-NEXT: s_addc_u32 s19, s19, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s21, 0
+; GFX10PLUS-NEXT: s_mul_i32 s21, s0, s14
+; GFX10PLUS-NEXT: s_addc_u32 s19, s19, s23
+; GFX10PLUS-NEXT: s_mul_i32 s23, s1, s13
+; GFX10PLUS-NEXT: s_cselect_b32 s20, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s21, s23, s21
+; GFX10PLUS-NEXT: s_mul_i32 s23, s2, s12
+; GFX10PLUS-NEXT: s_addc_u32 s22, s34, s22
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s2, s12
+; GFX10PLUS-NEXT: s_add_u32 s21, s23, s21
+; GFX10PLUS-NEXT: s_mul_i32 s23, s3, s11
+; GFX10PLUS-NEXT: s_addc_u32 s22, s34, s22
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s3, s11
+; GFX10PLUS-NEXT: s_add_u32 s21, s23, s21
+; GFX10PLUS-NEXT: s_mul_i32 s23, s4, s10
+; GFX10PLUS-NEXT: s_addc_u32 s22, s34, s22
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s4, s10
+; GFX10PLUS-NEXT: s_add_u32 s21, s23, s21
+; GFX10PLUS-NEXT: s_mul_i32 s23, s5, s9
+; GFX10PLUS-NEXT: s_addc_u32 s22, s34, s22
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s5, s9
+; GFX10PLUS-NEXT: s_add_u32 s21, s23, s21
+; GFX10PLUS-NEXT: s_mul_i32 s23, s6, s8
+; GFX10PLUS-NEXT: s_addc_u32 s22, s34, s22
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s6, s8
+; GFX10PLUS-NEXT: s_add_u32 s21, s23, s21
+; GFX10PLUS-NEXT: s_mul_i32 s23, s0, s13
+; GFX10PLUS-NEXT: s_addc_u32 s22, s34, s22
+; GFX10PLUS-NEXT: s_mul_hi_u32 s34, s0, s13
+; GFX10PLUS-NEXT: s_add_u32 s23, s23, s24
+; GFX10PLUS-NEXT: s_addc_u32 s21, s34, s21
+; GFX10PLUS-NEXT: s_mul_i32 s34, s1, s12
+; GFX10PLUS-NEXT: s_cselect_b32 s24, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s34, s23
+; GFX10PLUS-NEXT: s_addc_u32 s21, s35, s21
+; GFX10PLUS-NEXT: s_mul_i32 s35, s2, s11
+; GFX10PLUS-NEXT: s_cselect_b32 s34, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s35, s23
+; GFX10PLUS-NEXT: s_addc_u32 s21, s36, s21
+; GFX10PLUS-NEXT: s_mul_i32 s36, s3, s10
+; GFX10PLUS-NEXT: s_mul_hi_u32 s37, s3, s10
+; GFX10PLUS-NEXT: s_cselect_b32 s35, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s36, s23
+; GFX10PLUS-NEXT: s_addc_u32 s21, s37, s21
+; GFX10PLUS-NEXT: s_mul_i32 s37, s4, s9
+; GFX10PLUS-NEXT: s_mul_hi_u32 s38, s4, s9
+; GFX10PLUS-NEXT: s_cselect_b32 s36, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s37, s23
+; GFX10PLUS-NEXT: s_addc_u32 s21, s38, s21
+; GFX10PLUS-NEXT: s_mul_i32 s38, s5, s8
+; GFX10PLUS-NEXT: s_mul_hi_u32 s39, s5, s8
+; GFX10PLUS-NEXT: s_cselect_b32 s37, 1, 0
+; GFX10PLUS-NEXT: s_add_u32 s23, s38, s23
+; GFX10PLUS-NEXT: s_addc_u32 s21, s39, s21
+; GFX10PLUS-NEXT: s_cselect_b32 s38, 1, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s29, 0
+; GFX10PLUS-NEXT: s_mul_i32 s1, s1, s14
+; GFX10PLUS-NEXT: s_cselect_b32 s29, 1, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s30, 0
+; GFX10PLUS-NEXT: s_mul_i32 s2, s2, s13
+; GFX10PLUS-NEXT: s_addc_u32 s29, s29, 0
; GFX10PLUS-NEXT: s_cmp_lg_u32 s31, 0
-; GFX10PLUS-NEXT: s_mul_i32 s11, s4, s11
-; GFX10PLUS-NEXT: s_addc_u32 s22, s25, s22
-; GFX10PLUS-NEXT: s_mul_hi_u32 s25, s4, s10
-; GFX10PLUS-NEXT: s_addc_u32 s23, s26, s23
-; GFX10PLUS-NEXT: s_add_i32 s11, s25, s11
-; GFX10PLUS-NEXT: s_mul_i32 s25, s5, s10
-; GFX10PLUS-NEXT: s_mul_hi_u32 s26, s6, s8
-; GFX10PLUS-NEXT: s_add_i32 s11, s11, s25
-; GFX10PLUS-NEXT: s_mul_i32 s25, s6, s9
-; GFX10PLUS-NEXT: s_mul_i32 s4, s4, s10
-; GFX10PLUS-NEXT: s_add_i32 s10, s26, s25
-; GFX10PLUS-NEXT: s_mul_i32 s6, s6, s8
-; GFX10PLUS-NEXT: s_add_i32 s10, s10, s7
-; GFX10PLUS-NEXT: s_add_u32 s4, s6, s4
-; GFX10PLUS-NEXT: s_addc_u32 s6, s10, s11
-; GFX10PLUS-NEXT: s_add_u32 s7, s33, s34
-; GFX10PLUS-NEXT: s_mul_i32 s11, s9, s5
-; GFX10PLUS-NEXT: s_addc_u32 s10, 0, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s5, s9, s5
-; GFX10PLUS-NEXT: s_add_u32 s7, s11, s7
-; GFX10PLUS-NEXT: s_addc_u32 s5, s5, s10
-; GFX10PLUS-NEXT: s_add_u32 s4, s7, s4
-; GFX10PLUS-NEXT: s_mul_i32 s7, s0, s15
-; GFX10PLUS-NEXT: s_mul_hi_u32 s9, s0, s14
-; GFX10PLUS-NEXT: s_addc_u32 s5, s5, s6
-; GFX10PLUS-NEXT: s_add_i32 s6, s9, s7
-; GFX10PLUS-NEXT: s_mul_i32 s7, s1, s14
-; GFX10PLUS-NEXT: s_mul_hi_u32 s9, s2, s12
-; GFX10PLUS-NEXT: s_add_i32 s6, s6, s7
-; GFX10PLUS-NEXT: s_mul_i32 s7, s2, s13
; GFX10PLUS-NEXT: s_mul_i32 s3, s3, s12
-; GFX10PLUS-NEXT: s_add_i32 s7, s9, s7
-; GFX10PLUS-NEXT: s_mul_i32 s10, s0, s14
-; GFX10PLUS-NEXT: s_mul_i32 s2, s2, s12
-; GFX10PLUS-NEXT: s_add_i32 s7, s7, s3
-; GFX10PLUS-NEXT: s_add_u32 s2, s2, s10
-; GFX10PLUS-NEXT: s_addc_u32 s3, s7, s6
-; GFX10PLUS-NEXT: s_add_u32 s6, s35, s36
-; GFX10PLUS-NEXT: s_mul_i32 s9, s13, s1
-; GFX10PLUS-NEXT: s_addc_u32 s7, 0, 0
-; GFX10PLUS-NEXT: s_mul_hi_u32 s1, s13, s1
-; GFX10PLUS-NEXT: s_add_u32 s6, s9, s6
-; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s7
-; GFX10PLUS-NEXT: s_add_u32 s2, s6, s2
-; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s3
-; GFX10PLUS-NEXT: s_cmp_lg_u32 s21, 0
+; GFX10PLUS-NEXT: s_addc_u32 s29, s29, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s33, 0
+; GFX10PLUS-NEXT: s_mul_i32 s4, s4, s11
+; GFX10PLUS-NEXT: s_addc_u32 s29, s29, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s20, 0
+; GFX10PLUS-NEXT: s_mul_i32 s5, s5, s10
+; GFX10PLUS-NEXT: s_addc_u32 s20, s29, s23
+; GFX10PLUS-NEXT: s_cselect_b32 s23, 1, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s25, 0
+; GFX10PLUS-NEXT: s_mul_i32 s6, s6, s9
+; GFX10PLUS-NEXT: s_cselect_b32 s25, 1, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s26, 0
+; GFX10PLUS-NEXT: s_mul_i32 s26, s0, s15
+; GFX10PLUS-NEXT: s_addc_u32 s25, s25, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s27, 0
+; GFX10PLUS-NEXT: s_mul_i32 s7, s7, s8
+; GFX10PLUS-NEXT: s_addc_u32 s25, s25, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s28, 0
; GFX10PLUS-NEXT: s_mul_i32 s0, s0, s8
-; GFX10PLUS-NEXT: s_addc_u32 s2, s2, s4
-; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s5
-; GFX10PLUS-NEXT: s_cmp_lg_u32 s24, 0
-; GFX10PLUS-NEXT: s_mov_b32 s3, s18
-; GFX10PLUS-NEXT: s_addc_u32 s6, s22, s2
-; GFX10PLUS-NEXT: s_addc_u32 s7, s23, s1
-; GFX10PLUS-NEXT: s_mov_b32 s1, s16
+; GFX10PLUS-NEXT: s_addc_u32 s25, s25, 0
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s23, 0
+; GFX10PLUS-NEXT: s_addc_u32 s15, s25, s21
+; GFX10PLUS-NEXT: s_addc_u32 s21, s22, s26
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s38, 0
+; GFX10PLUS-NEXT: s_addc_u32 s1, s21, s1
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s37, 0
+; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s2
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s36, 0
; GFX10PLUS-NEXT: s_mov_b32 s2, s17
+; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s3
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s35, 0
+; GFX10PLUS-NEXT: s_mov_b32 s3, s18
+; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s4
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s34, 0
; GFX10PLUS-NEXT: s_mov_b32 s4, s19
+; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s5
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s24, 0
; GFX10PLUS-NEXT: s_mov_b32 s5, s20
+; GFX10PLUS-NEXT: s_addc_u32 s1, s1, s6
+; GFX10PLUS-NEXT: s_mov_b32 s6, s15
+; GFX10PLUS-NEXT: s_add_i32 s7, s1, s7
+; GFX10PLUS-NEXT: s_mov_b32 s1, s16
; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_mul_i256:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mov_b32 s23, 0
-; GFX12-NEXT: s_mov_b32 s22, s8
-; GFX12-NEXT: s_mov_b32 s20, s0
-; GFX12-NEXT: s_mov_b32 s21, s23
-; GFX12-NEXT: s_mov_b32 s24, s1
-; GFX12-NEXT: s_mul_u64 s[16:17], s[20:21], s[22:23]
-; GFX12-NEXT: s_mov_b32 s25, s23
-; GFX12-NEXT: s_mov_b32 s26, s17
-; GFX12-NEXT: s_mul_u64 s[18:19], s[24:25], s[22:23]
-; GFX12-NEXT: s_mov_b32 s27, s23
-; GFX12-NEXT: s_mov_b32 s28, s9
-; GFX12-NEXT: s_add_nc_u64 s[18:19], s[18:19], s[26:27]
-; GFX12-NEXT: s_mov_b32 s29, s23
-; GFX12-NEXT: s_mov_b32 s36, s2
-; GFX12-NEXT: s_mov_b32 s37, s23
-; GFX12-NEXT: s_mov_b32 s34, s19
-; GFX12-NEXT: s_mov_b32 s19, s23
-; GFX12-NEXT: s_mul_u64 s[26:27], s[20:21], s[28:29]
-; GFX12-NEXT: s_mov_b32 s30, s3
-; GFX12-NEXT: s_mul_u64 s[38:39], s[36:37], s[22:23]
-; GFX12-NEXT: s_mov_b32 s31, s23
-; GFX12-NEXT: s_add_nc_u64 s[26:27], s[26:27], s[18:19]
-; GFX12-NEXT: s_mul_u64 s[18:19], s[30:31], s[22:23]
-; GFX12-NEXT: s_mov_b32 s42, s39
-; GFX12-NEXT: s_mov_b32 s43, s23
-; GFX12-NEXT: s_mov_b32 s35, s23
-; GFX12-NEXT: s_mov_b32 s40, s27
-; GFX12-NEXT: s_mov_b32 s41, s23
-; GFX12-NEXT: s_add_nc_u64 s[18:19], s[18:19], s[42:43]
-; GFX12-NEXT: s_mul_u64 s[42:43], s[36:37], s[28:29]
-; GFX12-NEXT: s_add_nc_u64 s[34:35], s[34:35], s[40:41]
-; GFX12-NEXT: s_mov_b32 s40, s18
-; GFX12-NEXT: s_mov_b32 s46, s10
-; GFX12-NEXT: s_add_nc_u64 s[40:41], s[42:43], s[40:41]
-; GFX12-NEXT: s_mov_b32 s47, s23
-; GFX12-NEXT: s_mov_b32 s42, s23
-; GFX12-NEXT: s_mov_b32 s43, s40
-; GFX12-NEXT: s_mul_u64 s[48:49], s[20:21], s[46:47]
-; GFX12-NEXT: s_mov_b32 s39, s23
-; GFX12-NEXT: s_mul_u64 s[44:45], s[24:25], s[28:29]
-; GFX12-NEXT: s_or_b64 s[38:39], s[38:39], s[42:43]
-; GFX12-NEXT: s_mul_u64 s[42:43], s[24:25], s[46:47]
-; GFX12-NEXT: s_mov_b32 s50, s49
-; GFX12-NEXT: s_mov_b32 s51, s23
-; GFX12-NEXT: s_add_nc_u64 s[44:45], s[44:45], s[34:35]
-; GFX12-NEXT: s_mov_b32 s34, s11
-; GFX12-NEXT: s_mov_b32 s35, s23
-; GFX12-NEXT: s_add_nc_u64 s[42:43], s[42:43], s[50:51]
-; GFX12-NEXT: s_mul_u64 s[50:51], s[20:21], s[34:35]
-; GFX12-NEXT: s_mov_b32 s52, s42
-; GFX12-NEXT: s_mov_b32 s53, s23
-; GFX12-NEXT: s_add_co_u32 s17, s38, s44
-; GFX12-NEXT: s_add_co_ci_u32 s18, s39, s45
-; GFX12-NEXT: s_add_nc_u64 s[38:39], s[50:51], s[52:53]
-; GFX12-NEXT: s_mov_b32 s44, s23
-; GFX12-NEXT: s_mov_b32 s45, s38
-; GFX12-NEXT: s_mov_b32 s49, s23
-; GFX12-NEXT: s_cselect_b32 s27, -1, 0
-; GFX12-NEXT: s_or_b64 s[44:45], s[48:49], s[44:45]
-; GFX12-NEXT: s_mov_b32 s42, s43
-; GFX12-NEXT: s_mov_b32 s43, s23
-; GFX12-NEXT: s_mov_b32 s38, s39
-; GFX12-NEXT: s_mov_b32 s39, s23
-; GFX12-NEXT: s_mul_u64 s[48:49], s[24:25], s[34:35]
-; GFX12-NEXT: s_add_co_u32 s17, s44, s17
-; GFX12-NEXT: s_add_nc_u64 s[38:39], s[42:43], s[38:39]
-; GFX12-NEXT: s_add_co_ci_u32 s18, s45, s18
-; GFX12-NEXT: s_add_co_ci_u32 s33, s48, s38
-; GFX12-NEXT: s_add_co_ci_u32 s48, s49, s39
-; GFX12-NEXT: s_mov_b32 s38, s19
-; GFX12-NEXT: s_mov_b32 s39, s23
-; GFX12-NEXT: s_mov_b32 s40, s41
-; GFX12-NEXT: s_mov_b32 s41, s23
-; GFX12-NEXT: s_mul_u64 s[44:45], s[36:37], s[46:47]
-; GFX12-NEXT: s_mul_u64 s[42:43], s[30:31], s[28:29]
-; GFX12-NEXT: s_add_nc_u64 s[38:39], s[38:39], s[40:41]
-; GFX12-NEXT: s_cmp_lg_u32 s27, 0
-; GFX12-NEXT: s_mul_u64 s[40:41], s[30:31], s[46:47]
-; GFX12-NEXT: s_mov_b32 s46, s45
-; GFX12-NEXT: s_add_co_ci_u32 s19, s42, s38
-; GFX12-NEXT: s_add_co_ci_u32 s27, s43, s39
-; GFX12-NEXT: s_add_nc_u64 s[38:39], s[40:41], s[46:47]
-; GFX12-NEXT: s_mul_u64 s[36:37], s[36:37], s[34:35]
-; GFX12-NEXT: s_mov_b32 s40, s38
-; GFX12-NEXT: s_mov_b32 s41, s23
-; GFX12-NEXT: s_mov_b32 s42, s4
-; GFX12-NEXT: s_add_nc_u64 s[36:37], s[36:37], s[40:41]
-; GFX12-NEXT: s_mov_b32 s43, s23
-; GFX12-NEXT: s_add_co_u32 s19, s19, s33
-; GFX12-NEXT: s_mov_b32 s40, s23
-; GFX12-NEXT: s_mov_b32 s41, s36
-; GFX12-NEXT: s_mov_b32 s45, s23
-; GFX12-NEXT: s_add_co_ci_u32 s27, s27, s48
-; GFX12-NEXT: s_mul_u64 s[46:47], s[22:23], s[42:43]
-; GFX12-NEXT: s_cselect_b32 s33, -1, 0
-; GFX12-NEXT: s_or_b64 s[40:41], s[44:45], s[40:41]
-; GFX12-NEXT: s_mul_u64 s[42:43], s[28:29], s[42:43]
-; GFX12-NEXT: s_mov_b32 s44, s47
-; GFX12-NEXT: s_add_co_u32 s19, s40, s19
-; GFX12-NEXT: s_add_co_ci_u32 s27, s41, s27
-; GFX12-NEXT: s_add_nc_u64 s[40:41], s[42:43], s[44:45]
-; GFX12-NEXT: s_mov_b32 s44, s5
-; GFX12-NEXT: s_mov_b32 s48, s12
-; GFX12-NEXT: s_mov_b32 s49, s23
-; GFX12-NEXT: s_mov_b32 s42, s40
-; GFX12-NEXT: s_mul_u64 s[50:51], s[22:23], s[44:45]
-; GFX12-NEXT: s_mul_u64 s[52:53], s[48:49], s[20:21]
-; GFX12-NEXT: s_mov_b32 s22, s13
-; GFX12-NEXT: s_mov_b32 s43, s23
-; GFX12-NEXT: s_mul_u64 s[20:21], s[22:23], s[20:21]
-; GFX12-NEXT: s_add_nc_u64 s[42:43], s[50:51], s[42:43]
-; GFX12-NEXT: s_mov_b32 s50, s53
-; GFX12-NEXT: s_mov_b32 s51, s23
-; GFX12-NEXT: s_mul_u64 s[48:49], s[48:49], s[24:25]
-; GFX12-NEXT: s_add_nc_u64 s[20:21], s[20:21], s[50:51]
-; GFX12-NEXT: s_mov_b32 s54, s23
-; GFX12-NEXT: s_mov_b32 s50, s20
-; GFX12-NEXT: s_mov_b32 s55, s42
-; GFX12-NEXT: s_add_nc_u64 s[48:49], s[48:49], s[50:51]
-; GFX12-NEXT: s_mov_b32 s47, s23
-; GFX12-NEXT: s_mov_b32 s50, s23
-; GFX12-NEXT: s_mov_b32 s51, s48
-; GFX12-NEXT: s_mov_b32 s53, s23
-; GFX12-NEXT: s_cselect_b32 s56, -1, 0
-; GFX12-NEXT: s_or_b64 s[46:47], s[46:47], s[54:55]
-; GFX12-NEXT: s_or_b64 s[50:51], s[52:53], s[50:51]
-; GFX12-NEXT: s_mov_b32 s38, s39
-; GFX12-NEXT: s_add_co_u32 s20, s50, s46
-; GFX12-NEXT: s_add_co_ci_u32 s36, s51, s47
-; GFX12-NEXT: s_cselect_b32 s40, -1, 0
-; GFX12-NEXT: s_add_co_u32 s19, s19, s20
-; GFX12-NEXT: s_add_co_ci_u32 s20, s27, s36
-; GFX12-NEXT: s_mov_b32 s39, s23
-; GFX12-NEXT: s_mov_b32 s36, s37
-; GFX12-NEXT: s_mov_b32 s37, s23
-; GFX12-NEXT: s_mul_u64 s[30:31], s[30:31], s[34:35]
-; GFX12-NEXT: s_add_nc_u64 s[34:35], s[38:39], s[36:37]
-; GFX12-NEXT: s_mov_b32 s38, s43
-; GFX12-NEXT: s_add_nc_u64 s[30:31], s[30:31], s[34:35]
-; GFX12-NEXT: s_mov_b32 s34, s41
-; GFX12-NEXT: s_mov_b32 s35, s23
-; GFX12-NEXT: s_mul_u64 s[4:5], s[4:5], s[10:11]
-; GFX12-NEXT: s_mul_u64 s[6:7], s[6:7], s[8:9]
-; GFX12-NEXT: s_mov_b32 s8, s21
-; GFX12-NEXT: s_mov_b32 s9, s23
-; GFX12-NEXT: s_mov_b32 s10, s49
-; GFX12-NEXT: s_mov_b32 s11, s23
-; GFX12-NEXT: s_cselect_b32 s27, -1, 0
-; GFX12-NEXT: s_and_b32 s33, s33, exec_lo
-; GFX12-NEXT: s_cselect_b64 s[36:37], 1, 0
-; GFX12-NEXT: s_cmp_lg_u32 s56, 0
-; GFX12-NEXT: s_mul_u64 s[28:29], s[28:29], s[44:45]
-; GFX12-NEXT: s_add_nc_u64 s[34:35], s[34:35], s[38:39]
-; GFX12-NEXT: s_mul_u64 s[22:23], s[22:23], s[24:25]
-; GFX12-NEXT: s_add_nc_u64 s[4:5], s[6:7], s[4:5]
-; GFX12-NEXT: s_add_nc_u64 s[6:7], s[8:9], s[10:11]
-; GFX12-NEXT: s_mul_u64 s[0:1], s[0:1], s[14:15]
-; GFX12-NEXT: s_mul_u64 s[2:3], s[2:3], s[12:13]
-; GFX12-NEXT: s_add_co_ci_u32 s30, s30, s36
-; GFX12-NEXT: s_add_nc_u64 s[28:29], s[28:29], s[34:35]
-; GFX12-NEXT: s_add_nc_u64 s[6:7], s[22:23], s[6:7]
-; GFX12-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
-; GFX12-NEXT: s_add_co_ci_u32 s31, s31, s37
-; GFX12-NEXT: s_add_nc_u64 s[2:3], s[28:29], s[4:5]
-; GFX12-NEXT: s_add_nc_u64 s[0:1], s[6:7], s[0:1]
-; GFX12-NEXT: s_cmp_lg_u32 s40, 0
-; GFX12-NEXT: s_mov_b32 s4, s19
-; GFX12-NEXT: s_add_co_ci_u32 s0, s0, s2
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX12-NEXT: s_mul_i32 s17, s0, s10
+; GFX12-NEXT: s_mul_i32 s19, s1, s9
+; GFX12-NEXT: s_mul_hi_u32 s18, s0, s10
+; GFX12-NEXT: s_mul_hi_u32 s20, s1, s9
+; GFX12-NEXT: s_add_co_u32 s17, s19, s17
+; GFX12-NEXT: s_add_co_ci_u32 s18, s20, s18
+; GFX12-NEXT: s_mul_i32 s20, s2, s8
+; GFX12-NEXT: s_mul_hi_u32 s21, s2, s8
+; GFX12-NEXT: s_cselect_b32 s19, 1, 0
+; GFX12-NEXT: s_add_co_u32 s17, s20, s17
+; GFX12-NEXT: s_mul_hi_u32 s16, s0, s8
+; GFX12-NEXT: s_add_co_ci_u32 s18, s21, s18
+; GFX12-NEXT: s_mul_i32 s21, s0, s9
+; GFX12-NEXT: s_mul_hi_u32 s22, s0, s9
+; GFX12-NEXT: s_cselect_b32 s20, 1, 0
+; GFX12-NEXT: s_add_co_u32 s16, s21, s16
+; GFX12-NEXT: s_add_co_ci_u32 s17, s22, s17
+; GFX12-NEXT: s_mul_i32 s22, s1, s8
+; GFX12-NEXT: s_mul_hi_u32 s23, s1, s8
+; GFX12-NEXT: s_cselect_b32 s21, 1, 0
+; GFX12-NEXT: s_add_co_u32 s16, s22, s16
+; GFX12-NEXT: s_add_co_ci_u32 s17, s23, s17
+; GFX12-NEXT: s_mul_i32 s23, s0, s12
+; GFX12-NEXT: s_mul_i32 s25, s1, s11
+; GFX12-NEXT: s_mul_hi_u32 s24, s0, s12
+; GFX12-NEXT: s_mul_hi_u32 s26, s1, s11
+; GFX12-NEXT: s_cselect_b32 s22, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s25, s23
+; GFX12-NEXT: s_add_co_ci_u32 s24, s26, s24
+; GFX12-NEXT: s_mul_i32 s26, s2, s10
+; GFX12-NEXT: s_mul_hi_u32 s27, s2, s10
+; GFX12-NEXT: s_cselect_b32 s25, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s26, s23
+; GFX12-NEXT: s_add_co_ci_u32 s24, s27, s24
+; GFX12-NEXT: s_mul_i32 s27, s3, s9
+; GFX12-NEXT: s_mul_hi_u32 s28, s3, s9
+; GFX12-NEXT: s_cselect_b32 s26, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s27, s23
+; GFX12-NEXT: s_add_co_ci_u32 s24, s28, s24
+; GFX12-NEXT: s_mul_i32 s28, s4, s8
+; GFX12-NEXT: s_mul_hi_u32 s29, s4, s8
+; GFX12-NEXT: s_cselect_b32 s27, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s28, s23
+; GFX12-NEXT: s_add_co_ci_u32 s24, s29, s24
+; GFX12-NEXT: s_mul_i32 s29, s0, s11
+; GFX12-NEXT: s_mul_hi_u32 s30, s0, s11
+; GFX12-NEXT: s_cselect_b32 s28, 1, 0
+; GFX12-NEXT: s_add_co_u32 s18, s29, s18
+; GFX12-NEXT: s_add_co_ci_u32 s23, s30, s23
+; GFX12-NEXT: s_mul_i32 s30, s1, s10
+; GFX12-NEXT: s_mul_hi_u32 s31, s1, s10
+; GFX12-NEXT: s_cselect_b32 s29, 1, 0
+; GFX12-NEXT: s_add_co_u32 s18, s30, s18
+; GFX12-NEXT: s_add_co_ci_u32 s23, s31, s23
+; GFX12-NEXT: s_mul_i32 s31, s2, s9
+; GFX12-NEXT: s_mul_hi_u32 s33, s2, s9
+; GFX12-NEXT: s_cselect_b32 s30, 1, 0
+; GFX12-NEXT: s_add_co_u32 s18, s31, s18
+; GFX12-NEXT: s_add_co_ci_u32 s23, s33, s23
+; GFX12-NEXT: s_mul_i32 s33, s3, s8
+; GFX12-NEXT: s_mul_hi_u32 s34, s3, s8
+; GFX12-NEXT: s_cselect_b32 s31, 1, 0
+; GFX12-NEXT: s_add_co_u32 s18, s33, s18
+; GFX12-NEXT: s_add_co_ci_u32 s23, s34, s23
+; GFX12-NEXT: s_cselect_b32 s33, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s21, 0
+; GFX12-NEXT: s_mul_hi_u32 s34, s1, s13
+; GFX12-NEXT: s_cselect_b32 s21, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s22, 0
+; GFX12-NEXT: s_mul_hi_u32 s22, s0, s14
+; GFX12-NEXT: s_add_co_ci_u32 s18, s21, s18
+; GFX12-NEXT: s_cselect_b32 s21, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s19, 0
+; GFX12-NEXT: s_mul_hi_u32 s35, s1, s12
+; GFX12-NEXT: s_cselect_b32 s19, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s20, 0
+; GFX12-NEXT: s_mul_hi_u32 s36, s2, s11
+; GFX12-NEXT: s_add_co_ci_u32 s19, s19, 0
+; GFX12-NEXT: s_cmp_lg_u32 s21, 0
+; GFX12-NEXT: s_mul_i32 s21, s0, s14
+; GFX12-NEXT: s_add_co_ci_u32 s19, s19, s23
+; GFX12-NEXT: s_mul_i32 s23, s1, s13
+; GFX12-NEXT: s_cselect_b32 s20, 1, 0
+; GFX12-NEXT: s_add_co_u32 s21, s23, s21
+; GFX12-NEXT: s_mul_i32 s23, s2, s12
+; GFX12-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX12-NEXT: s_mul_hi_u32 s34, s2, s12
+; GFX12-NEXT: s_add_co_u32 s21, s23, s21
+; GFX12-NEXT: s_mul_i32 s23, s3, s11
+; GFX12-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX12-NEXT: s_mul_hi_u32 s34, s3, s11
+; GFX12-NEXT: s_add_co_u32 s21, s23, s21
+; GFX12-NEXT: s_mul_i32 s23, s4, s10
+; GFX12-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX12-NEXT: s_mul_hi_u32 s34, s4, s10
+; GFX12-NEXT: s_add_co_u32 s21, s23, s21
+; GFX12-NEXT: s_mul_i32 s23, s5, s9
+; GFX12-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX12-NEXT: s_mul_hi_u32 s34, s5, s9
+; GFX12-NEXT: s_add_co_u32 s21, s23, s21
+; GFX12-NEXT: s_mul_i32 s23, s6, s8
+; GFX12-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX12-NEXT: s_mul_hi_u32 s34, s6, s8
+; GFX12-NEXT: s_add_co_u32 s21, s23, s21
+; GFX12-NEXT: s_mul_i32 s23, s0, s13
+; GFX12-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX12-NEXT: s_mul_hi_u32 s34, s0, s13
+; GFX12-NEXT: s_add_co_u32 s23, s23, s24
+; GFX12-NEXT: s_add_co_ci_u32 s21, s34, s21
+; GFX12-NEXT: s_mul_i32 s34, s1, s12
+; GFX12-NEXT: s_cselect_b32 s24, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s34, s23
+; GFX12-NEXT: s_add_co_ci_u32 s21, s35, s21
+; GFX12-NEXT: s_mul_i32 s35, s2, s11
+; GFX12-NEXT: s_cselect_b32 s34, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s35, s23
+; GFX12-NEXT: s_add_co_ci_u32 s21, s36, s21
+; GFX12-NEXT: s_mul_i32 s36, s3, s10
+; GFX12-NEXT: s_mul_hi_u32 s37, s3, s10
+; GFX12-NEXT: s_cselect_b32 s35, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s36, s23
+; GFX12-NEXT: s_add_co_ci_u32 s21, s37, s21
+; GFX12-NEXT: s_mul_i32 s37, s4, s9
+; GFX12-NEXT: s_mul_hi_u32 s38, s4, s9
+; GFX12-NEXT: s_cselect_b32 s36, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s37, s23
+; GFX12-NEXT: s_add_co_ci_u32 s21, s38, s21
+; GFX12-NEXT: s_mul_i32 s38, s5, s8
+; GFX12-NEXT: s_mul_hi_u32 s39, s5, s8
+; GFX12-NEXT: s_cselect_b32 s37, 1, 0
+; GFX12-NEXT: s_add_co_u32 s23, s38, s23
+; GFX12-NEXT: s_add_co_ci_u32 s21, s39, s21
+; GFX12-NEXT: s_cselect_b32 s38, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s29, 0
+; GFX12-NEXT: s_mul_i32 s1, s1, s14
+; GFX12-NEXT: s_cselect_b32 s29, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s30, 0
+; GFX12-NEXT: s_mul_i32 s2, s2, s13
+; GFX12-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX12-NEXT: s_cmp_lg_u32 s31, 0
+; GFX12-NEXT: s_mul_i32 s3, s3, s12
+; GFX12-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX12-NEXT: s_cmp_lg_u32 s33, 0
+; GFX12-NEXT: s_mul_i32 s4, s4, s11
+; GFX12-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX12-NEXT: s_cmp_lg_u32 s20, 0
+; GFX12-NEXT: s_mul_i32 s5, s5, s10
+; GFX12-NEXT: s_add_co_ci_u32 s20, s29, s23
+; GFX12-NEXT: s_cselect_b32 s23, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s25, 0
+; GFX12-NEXT: s_mul_i32 s6, s6, s9
+; GFX12-NEXT: s_cselect_b32 s25, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s26, 0
+; GFX12-NEXT: s_mul_i32 s26, s0, s15
+; GFX12-NEXT: s_add_co_ci_u32 s25, s25, 0
; GFX12-NEXT: s_cmp_lg_u32 s27, 0
+; GFX12-NEXT: s_mul_i32 s7, s7, s8
+; GFX12-NEXT: s_add_co_ci_u32 s25, s25, 0
+; GFX12-NEXT: s_cmp_lg_u32 s28, 0
+; GFX12-NEXT: s_mul_i32 s0, s0, s8
+; GFX12-NEXT: s_add_co_ci_u32 s25, s25, 0
+; GFX12-NEXT: s_cmp_lg_u32 s23, 0
+; GFX12-NEXT: s_add_co_ci_u32 s15, s25, s21
+; GFX12-NEXT: s_add_co_ci_u32 s21, s22, s26
+; GFX12-NEXT: s_cmp_lg_u32 s38, 0
+; GFX12-NEXT: s_add_co_ci_u32 s1, s21, s1
+; GFX12-NEXT: s_cmp_lg_u32 s37, 0
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s2
+; GFX12-NEXT: s_cmp_lg_u32 s36, 0
; GFX12-NEXT: s_mov_b32 s2, s17
-; GFX12-NEXT: s_add_co_ci_u32 s6, s30, s0
-; GFX12-NEXT: s_add_co_ci_u32 s7, s31, s1
-; GFX12-NEXT: s_mov_b32 s0, s16
-; GFX12-NEXT: s_mov_b32 s1, s26
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX12-NEXT: s_cmp_lg_u32 s35, 0
; GFX12-NEXT: s_mov_b32 s3, s18
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s4
+; GFX12-NEXT: s_cmp_lg_u32 s34, 0
+; GFX12-NEXT: s_mov_b32 s4, s19
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s5
+; GFX12-NEXT: s_cmp_lg_u32 s24, 0
; GFX12-NEXT: s_mov_b32 s5, s20
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s6
+; GFX12-NEXT: s_mov_b32 s6, s15
+; GFX12-NEXT: s_add_co_i32 s7, s1, s7
+; GFX12-NEXT: s_mov_b32 s1, s16
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: s_mul_i256:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[16:17], 0xffffffff
-; GFX1250-NEXT: s_mov_b32 s25, 0
-; GFX1250-NEXT: s_and_b64 s[30:31], s[8:9], s[16:17]
-; GFX1250-NEXT: s_mov_b32 s20, s0
-; GFX1250-NEXT: s_mov_b32 s21, s25
-; GFX1250-NEXT: s_mov_b32 s24, s1
-; GFX1250-NEXT: s_mul_u64 s[16:17], s[20:21], s[30:31]
-; GFX1250-NEXT: s_mov_b32 s36, s2
-; GFX1250-NEXT: s_mov_b32 s37, s25
-; GFX1250-NEXT: s_mul_u64 s[18:19], s[24:25], s[30:31]
-; GFX1250-NEXT: s_mov_b32 s22, s17
-; GFX1250-NEXT: s_mov_b32 s23, s25
-; GFX1250-NEXT: s_mov_b32 s28, s3
-; GFX1250-NEXT: s_mov_b32 s29, s25
-; GFX1250-NEXT: s_mul_u64 s[38:39], s[36:37], s[30:31]
-; GFX1250-NEXT: s_add_nc_u64 s[18:19], s[18:19], s[22:23]
-; GFX1250-NEXT: s_mov_b32 s26, s9
-; GFX1250-NEXT: s_mov_b32 s27, s25
-; GFX1250-NEXT: s_mul_u64 s[40:41], s[28:29], s[30:31]
-; GFX1250-NEXT: s_mov_b32 s42, s39
-; GFX1250-NEXT: s_mov_b32 s43, s25
-; GFX1250-NEXT: s_mov_b32 s34, s19
-; GFX1250-NEXT: s_mov_b32 s19, s25
-; GFX1250-NEXT: s_mul_u64 s[22:23], s[20:21], s[26:27]
-; GFX1250-NEXT: s_add_nc_u64 s[40:41], s[40:41], s[42:43]
-; GFX1250-NEXT: s_add_nc_u64 s[22:23], s[22:23], s[18:19]
-; GFX1250-NEXT: s_mov_b32 s42, s40
-; GFX1250-NEXT: s_mul_u64 s[44:45], s[36:37], s[26:27]
-; GFX1250-NEXT: s_mov_b32 s35, s25
-; GFX1250-NEXT: s_mov_b32 s18, s23
-; GFX1250-NEXT: s_add_nc_u64 s[42:43], s[44:45], s[42:43]
-; GFX1250-NEXT: s_mov_b32 s44, s10
-; GFX1250-NEXT: s_mov_b32 s45, s25
-; GFX1250-NEXT: s_add_nc_u64 s[18:19], s[34:35], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[34:35], s[24:25], s[26:27]
-; GFX1250-NEXT: s_mul_u64 s[46:47], s[20:21], s[44:45]
-; GFX1250-NEXT: s_add_nc_u64 s[18:19], s[34:35], s[18:19]
-; GFX1250-NEXT: s_mov_b32 s34, s25
-; GFX1250-NEXT: s_mov_b32 s35, s42
-; GFX1250-NEXT: s_mov_b32 s39, s25
-; GFX1250-NEXT: s_mul_u64 s[48:49], s[24:25], s[44:45]
-; GFX1250-NEXT: s_mov_b32 s50, s47
-; GFX1250-NEXT: s_mov_b32 s51, s25
-; GFX1250-NEXT: s_or_b64 s[38:39], s[38:39], s[34:35]
-; GFX1250-NEXT: s_add_nc_u64 s[48:49], s[48:49], s[50:51]
-; GFX1250-NEXT: s_mov_b32 s34, s11
-; GFX1250-NEXT: s_mov_b32 s35, s25
-; GFX1250-NEXT: s_mov_b32 s50, s48
-; GFX1250-NEXT: s_mul_u64 s[52:53], s[20:21], s[34:35]
-; GFX1250-NEXT: s_add_co_u32 s17, s38, s18
-; GFX1250-NEXT: s_add_co_ci_u32 s23, s39, s19
-; GFX1250-NEXT: s_add_nc_u64 s[18:19], s[52:53], s[50:51]
-; GFX1250-NEXT: s_mov_b32 s38, s25
-; GFX1250-NEXT: s_mov_b32 s39, s18
-; GFX1250-NEXT: s_mov_b32 s47, s25
-; GFX1250-NEXT: s_cselect_b32 s33, -1, 0
-; GFX1250-NEXT: s_or_b64 s[38:39], s[46:47], s[38:39]
-; GFX1250-NEXT: s_mov_b32 s46, s49
-; GFX1250-NEXT: s_mov_b32 s18, s19
-; GFX1250-NEXT: s_mov_b32 s19, s25
-; GFX1250-NEXT: s_add_co_u32 s17, s38, s17
-; GFX1250-NEXT: s_add_nc_u64 s[46:47], s[46:47], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[48:49], s[24:25], s[34:35]
-; GFX1250-NEXT: s_add_co_ci_u32 s18, s39, s23
-; GFX1250-NEXT: s_add_co_ci_u32 s19, s48, s46
-; GFX1250-NEXT: s_mov_b32 s38, s41
-; GFX1250-NEXT: s_mov_b32 s39, s25
-; GFX1250-NEXT: s_mov_b32 s40, s43
-; GFX1250-NEXT: s_mov_b32 s41, s25
-; GFX1250-NEXT: s_add_co_ci_u32 s23, s49, s47
-; GFX1250-NEXT: s_add_nc_u64 s[38:39], s[38:39], s[40:41]
-; GFX1250-NEXT: s_mul_u64 s[40:41], s[28:29], s[26:27]
+; GFX1250-NEXT: s_mul_i32 s17, s0, s10
+; GFX1250-NEXT: s_mul_i32 s19, s1, s9
+; GFX1250-NEXT: s_mul_hi_u32 s18, s0, s10
+; GFX1250-NEXT: s_mul_hi_u32 s20, s1, s9
+; GFX1250-NEXT: s_add_co_u32 s17, s19, s17
+; GFX1250-NEXT: s_add_co_ci_u32 s18, s20, s18
+; GFX1250-NEXT: s_mul_i32 s20, s2, s8
+; GFX1250-NEXT: s_mul_hi_u32 s21, s2, s8
+; GFX1250-NEXT: s_cselect_b32 s19, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s17, s20, s17
+; GFX1250-NEXT: s_mul_hi_u32 s16, s0, s8
+; GFX1250-NEXT: s_add_co_ci_u32 s18, s21, s18
+; GFX1250-NEXT: s_mul_i32 s21, s0, s9
+; GFX1250-NEXT: s_mul_hi_u32 s22, s0, s9
+; GFX1250-NEXT: s_cselect_b32 s20, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s16, s21, s16
+; GFX1250-NEXT: s_add_co_ci_u32 s17, s22, s17
+; GFX1250-NEXT: s_mul_i32 s22, s1, s8
+; GFX1250-NEXT: s_mul_hi_u32 s23, s1, s8
+; GFX1250-NEXT: s_cselect_b32 s21, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s16, s22, s16
+; GFX1250-NEXT: s_add_co_ci_u32 s17, s23, s17
+; GFX1250-NEXT: s_mul_i32 s23, s0, s12
+; GFX1250-NEXT: s_mul_i32 s25, s1, s11
+; GFX1250-NEXT: s_mul_hi_u32 s24, s0, s12
+; GFX1250-NEXT: s_mul_hi_u32 s26, s1, s11
+; GFX1250-NEXT: s_cselect_b32 s22, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s25, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s24, s26, s24
+; GFX1250-NEXT: s_mul_i32 s26, s2, s10
+; GFX1250-NEXT: s_mul_hi_u32 s27, s2, s10
+; GFX1250-NEXT: s_cselect_b32 s25, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s26, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s24, s27, s24
+; GFX1250-NEXT: s_mul_i32 s27, s3, s9
+; GFX1250-NEXT: s_mul_hi_u32 s28, s3, s9
+; GFX1250-NEXT: s_cselect_b32 s26, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s27, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s24, s28, s24
+; GFX1250-NEXT: s_mul_i32 s28, s4, s8
+; GFX1250-NEXT: s_mul_hi_u32 s29, s4, s8
+; GFX1250-NEXT: s_cselect_b32 s27, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s28, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s24, s29, s24
+; GFX1250-NEXT: s_mul_i32 s29, s0, s11
+; GFX1250-NEXT: s_mul_hi_u32 s30, s0, s11
+; GFX1250-NEXT: s_cselect_b32 s28, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s18, s29, s18
+; GFX1250-NEXT: s_add_co_ci_u32 s23, s30, s23
+; GFX1250-NEXT: s_mul_i32 s30, s1, s10
+; GFX1250-NEXT: s_mul_hi_u32 s31, s1, s10
+; GFX1250-NEXT: s_cselect_b32 s29, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s18, s30, s18
+; GFX1250-NEXT: s_add_co_ci_u32 s23, s31, s23
+; GFX1250-NEXT: s_mul_i32 s31, s2, s9
+; GFX1250-NEXT: s_mul_hi_u32 s33, s2, s9
+; GFX1250-NEXT: s_cselect_b32 s30, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s18, s31, s18
+; GFX1250-NEXT: s_add_co_ci_u32 s23, s33, s23
+; GFX1250-NEXT: s_mul_i32 s33, s3, s8
+; GFX1250-NEXT: s_mul_hi_u32 s34, s3, s8
+; GFX1250-NEXT: s_cselect_b32 s31, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s18, s33, s18
+; GFX1250-NEXT: s_add_co_ci_u32 s23, s34, s23
+; GFX1250-NEXT: s_cselect_b32 s33, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s21, 0
+; GFX1250-NEXT: s_mul_hi_u32 s34, s1, s13
+; GFX1250-NEXT: s_cselect_b32 s21, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s22, 0
+; GFX1250-NEXT: s_mul_hi_u32 s22, s0, s14
+; GFX1250-NEXT: s_add_co_ci_u32 s18, s21, s18
+; GFX1250-NEXT: s_cselect_b32 s21, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s19, 0
+; GFX1250-NEXT: s_mul_hi_u32 s35, s1, s12
+; GFX1250-NEXT: s_cselect_b32 s19, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s20, 0
+; GFX1250-NEXT: s_mul_hi_u32 s36, s2, s11
+; GFX1250-NEXT: s_add_co_ci_u32 s19, s19, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s21, 0
+; GFX1250-NEXT: s_mul_i32 s21, s0, s14
+; GFX1250-NEXT: s_add_co_ci_u32 s19, s19, s23
+; GFX1250-NEXT: s_mul_i32 s23, s1, s13
+; GFX1250-NEXT: s_cselect_b32 s20, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s21, s23, s21
+; GFX1250-NEXT: s_mul_i32 s23, s2, s12
+; GFX1250-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX1250-NEXT: s_mul_hi_u32 s34, s2, s12
+; GFX1250-NEXT: s_add_co_u32 s21, s23, s21
+; GFX1250-NEXT: s_mul_i32 s23, s3, s11
+; GFX1250-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX1250-NEXT: s_mul_hi_u32 s34, s3, s11
+; GFX1250-NEXT: s_add_co_u32 s21, s23, s21
+; GFX1250-NEXT: s_mul_i32 s23, s4, s10
+; GFX1250-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX1250-NEXT: s_mul_hi_u32 s34, s4, s10
+; GFX1250-NEXT: s_add_co_u32 s21, s23, s21
+; GFX1250-NEXT: s_mul_i32 s23, s5, s9
+; GFX1250-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX1250-NEXT: s_mul_hi_u32 s34, s5, s9
+; GFX1250-NEXT: s_add_co_u32 s21, s23, s21
+; GFX1250-NEXT: s_mul_i32 s23, s6, s8
+; GFX1250-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX1250-NEXT: s_mul_hi_u32 s34, s6, s8
+; GFX1250-NEXT: s_add_co_u32 s21, s23, s21
+; GFX1250-NEXT: s_mul_i32 s23, s0, s13
+; GFX1250-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX1250-NEXT: s_mul_hi_u32 s34, s0, s13
+; GFX1250-NEXT: s_add_co_u32 s23, s23, s24
+; GFX1250-NEXT: s_add_co_ci_u32 s21, s34, s21
+; GFX1250-NEXT: s_mul_i32 s34, s1, s12
+; GFX1250-NEXT: s_cselect_b32 s24, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s34, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s21, s35, s21
+; GFX1250-NEXT: s_mul_i32 s35, s2, s11
+; GFX1250-NEXT: s_cselect_b32 s34, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s35, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s21, s36, s21
+; GFX1250-NEXT: s_mul_i32 s36, s3, s10
+; GFX1250-NEXT: s_mul_hi_u32 s37, s3, s10
+; GFX1250-NEXT: s_cselect_b32 s35, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s36, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s21, s37, s21
+; GFX1250-NEXT: s_mul_i32 s37, s4, s9
+; GFX1250-NEXT: s_mul_hi_u32 s38, s4, s9
+; GFX1250-NEXT: s_cselect_b32 s36, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s37, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s21, s38, s21
+; GFX1250-NEXT: s_mul_i32 s38, s5, s8
+; GFX1250-NEXT: s_mul_hi_u32 s39, s5, s8
+; GFX1250-NEXT: s_cselect_b32 s37, 1, 0
+; GFX1250-NEXT: s_add_co_u32 s23, s38, s23
+; GFX1250-NEXT: s_add_co_ci_u32 s21, s39, s21
+; GFX1250-NEXT: s_cselect_b32 s38, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s29, 0
+; GFX1250-NEXT: s_mul_i32 s1, s1, s14
+; GFX1250-NEXT: s_cselect_b32 s29, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s30, 0
+; GFX1250-NEXT: s_mul_i32 s2, s2, s13
+; GFX1250-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s31, 0
+; GFX1250-NEXT: s_mul_i32 s3, s3, s12
+; GFX1250-NEXT: s_add_co_ci_u32 s29, s29, 0
; GFX1250-NEXT: s_cmp_lg_u32 s33, 0
-; GFX1250-NEXT: s_mov_b32 s43, s25
-; GFX1250-NEXT: s_add_co_ci_u32 s33, s40, s38
-; GFX1250-NEXT: s_add_co_ci_u32 s46, s41, s39
-; GFX1250-NEXT: s_mul_u64 s[38:39], s[36:37], s[44:45]
-; GFX1250-NEXT: s_mul_u64 s[40:41], s[28:29], s[44:45]
-; GFX1250-NEXT: s_mov_b32 s42, s39
-; GFX1250-NEXT: s_mul_u64 s[36:37], s[36:37], s[34:35]
-; GFX1250-NEXT: s_add_nc_u64 s[40:41], s[40:41], s[42:43]
-; GFX1250-NEXT: s_add_co_u32 s19, s33, s19
-; GFX1250-NEXT: s_mov_b32 s42, s40
-; GFX1250-NEXT: s_mov_b32 s39, s25
-; GFX1250-NEXT: s_add_nc_u64 s[36:37], s[36:37], s[42:43]
-; GFX1250-NEXT: s_mov_b32 s42, s25
-; GFX1250-NEXT: s_mov_b32 s43, s36
-; GFX1250-NEXT: s_mov_b32 s44, s4
-; GFX1250-NEXT: s_add_co_ci_u32 s23, s46, s23
-; GFX1250-NEXT: s_cselect_b32 s33, -1, 0
-; GFX1250-NEXT: s_or_b64 s[38:39], s[38:39], s[42:43]
-; GFX1250-NEXT: s_mul_u64 s[42:43], s[30:31], s[44:45]
-; GFX1250-NEXT: s_mul_u64 s[44:45], s[26:27], s[44:45]
-; GFX1250-NEXT: s_mov_b32 s46, s43
-; GFX1250-NEXT: s_mov_b32 s47, s25
-; GFX1250-NEXT: s_add_co_u32 s19, s38, s19
-; GFX1250-NEXT: s_add_nc_u64 s[44:45], s[44:45], s[46:47]
-; GFX1250-NEXT: s_mov_b32 s46, s5
-; GFX1250-NEXT: s_add_co_ci_u32 s23, s39, s23
-; GFX1250-NEXT: s_mov_b32 s38, s44
-; GFX1250-NEXT: s_mov_b32 s39, s25
-; GFX1250-NEXT: s_mul_u64 s[30:31], s[30:31], s[46:47]
-; GFX1250-NEXT: s_mov_b32 s48, s12
-; GFX1250-NEXT: s_mov_b32 s49, s25
-; GFX1250-NEXT: s_add_nc_u64 s[30:31], s[30:31], s[38:39]
-; GFX1250-NEXT: s_mov_b32 s38, s13
-; GFX1250-NEXT: s_mul_u64 s[50:51], s[48:49], s[20:21]
-; GFX1250-NEXT: s_mul_u64 s[20:21], s[38:39], s[20:21]
-; GFX1250-NEXT: s_mov_b32 s52, s51
-; GFX1250-NEXT: s_mov_b32 s53, s25
-; GFX1250-NEXT: s_mul_u64 s[48:49], s[48:49], s[24:25]
-; GFX1250-NEXT: s_add_nc_u64 s[20:21], s[20:21], s[52:53]
-; GFX1250-NEXT: s_mov_b32 s54, s25
-; GFX1250-NEXT: s_mov_b32 s52, s20
-; GFX1250-NEXT: s_mov_b32 s55, s30
-; GFX1250-NEXT: s_add_nc_u64 s[48:49], s[48:49], s[52:53]
-; GFX1250-NEXT: s_mov_b32 s43, s25
-; GFX1250-NEXT: s_mov_b32 s52, s25
-; GFX1250-NEXT: s_mov_b32 s53, s48
-; GFX1250-NEXT: s_mov_b32 s51, s25
-; GFX1250-NEXT: s_cselect_b32 s44, -1, 0
-; GFX1250-NEXT: s_or_b64 s[42:43], s[42:43], s[54:55]
-; GFX1250-NEXT: s_or_b64 s[50:51], s[50:51], s[52:53]
-; GFX1250-NEXT: s_mov_b32 s40, s41
-; GFX1250-NEXT: s_add_co_u32 s20, s50, s42
-; GFX1250-NEXT: s_add_co_ci_u32 s30, s51, s43
-; GFX1250-NEXT: s_mov_b32 s41, s25
-; GFX1250-NEXT: s_mov_b32 s36, s37
-; GFX1250-NEXT: s_mov_b32 s37, s25
-; GFX1250-NEXT: s_cselect_b32 s42, -1, 0
-; GFX1250-NEXT: s_add_co_u32 s19, s19, s20
-; GFX1250-NEXT: s_add_co_ci_u32 s20, s23, s30
-; GFX1250-NEXT: s_add_nc_u64 s[36:37], s[40:41], s[36:37]
-; GFX1250-NEXT: s_mul_u64 s[28:29], s[28:29], s[34:35]
-; GFX1250-NEXT: s_cselect_b32 s23, -1, 0
-; GFX1250-NEXT: s_and_b32 s30, s33, exec_lo
-; GFX1250-NEXT: s_add_nc_u64 s[28:29], s[28:29], s[36:37]
-; GFX1250-NEXT: s_mov_b32 s36, s45
-; GFX1250-NEXT: s_mov_b32 s37, s25
-; GFX1250-NEXT: s_mov_b32 s30, s31
-; GFX1250-NEXT: s_mov_b32 s31, s25
-; GFX1250-NEXT: s_mul_u64 s[4:5], s[4:5], s[10:11]
-; GFX1250-NEXT: s_mul_u64 s[6:7], s[6:7], s[8:9]
-; GFX1250-NEXT: s_mov_b32 s8, s21
-; GFX1250-NEXT: s_mov_b32 s9, s25
-; GFX1250-NEXT: s_mov_b32 s10, s49
-; GFX1250-NEXT: s_mov_b32 s11, s25
-; GFX1250-NEXT: s_cselect_b64 s[34:35], 1, 0
-; GFX1250-NEXT: s_cmp_lg_u32 s44, 0
-; GFX1250-NEXT: s_add_nc_u64 s[30:31], s[36:37], s[30:31]
-; GFX1250-NEXT: s_mul_u64 s[26:27], s[26:27], s[46:47]
-; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[6:7], s[4:5]
-; GFX1250-NEXT: s_add_nc_u64 s[6:7], s[8:9], s[10:11]
-; GFX1250-NEXT: s_mul_u64 s[8:9], s[38:39], s[24:25]
-; GFX1250-NEXT: s_mul_u64 s[0:1], s[0:1], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[2:3], s[2:3], s[12:13]
-; GFX1250-NEXT: s_add_co_ci_u32 s28, s28, s34
-; GFX1250-NEXT: s_add_nc_u64 s[26:27], s[26:27], s[30:31]
-; GFX1250-NEXT: s_add_nc_u64 s[6:7], s[8:9], s[6:7]
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
-; GFX1250-NEXT: s_add_co_ci_u32 s29, s29, s35
-; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[26:27], s[4:5]
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[6:7], s[0:1]
-; GFX1250-NEXT: s_cmp_lg_u32 s42, 0
-; GFX1250-NEXT: s_mov_b32 s4, s19
-; GFX1250-NEXT: s_add_co_ci_u32 s0, s0, s2
-; GFX1250-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX1250-NEXT: s_mul_i32 s4, s4, s11
+; GFX1250-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s20, 0
+; GFX1250-NEXT: s_mul_i32 s5, s5, s10
+; GFX1250-NEXT: s_add_co_ci_u32 s20, s29, s23
+; GFX1250-NEXT: s_cselect_b32 s23, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s25, 0
+; GFX1250-NEXT: s_mul_i32 s6, s6, s9
+; GFX1250-NEXT: s_cselect_b32 s25, 1, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s26, 0
+; GFX1250-NEXT: s_mul_i32 s26, s0, s15
+; GFX1250-NEXT: s_add_co_ci_u32 s25, s25, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s27, 0
+; GFX1250-NEXT: s_mul_i32 s7, s7, s8
+; GFX1250-NEXT: s_add_co_ci_u32 s25, s25, 0
+; GFX1250-NEXT: s_cmp_lg_u32 s28, 0
+; GFX1250-NEXT: s_mul_i32 s0, s0, s8
+; GFX1250-NEXT: s_add_co_ci_u32 s25, s25, 0
; GFX1250-NEXT: s_cmp_lg_u32 s23, 0
+; GFX1250-NEXT: s_add_co_ci_u32 s15, s25, s21
+; GFX1250-NEXT: s_add_co_ci_u32 s21, s22, s26
+; GFX1250-NEXT: s_cmp_lg_u32 s38, 0
+; GFX1250-NEXT: s_add_co_ci_u32 s1, s21, s1
+; GFX1250-NEXT: s_cmp_lg_u32 s37, 0
+; GFX1250-NEXT: s_add_co_ci_u32 s1, s1, s2
+; GFX1250-NEXT: s_cmp_lg_u32 s36, 0
; GFX1250-NEXT: s_mov_b32 s2, s17
-; GFX1250-NEXT: s_add_co_ci_u32 s6, s28, s0
-; GFX1250-NEXT: s_add_co_ci_u32 s7, s29, s1
-; GFX1250-NEXT: s_mov_b32 s0, s16
-; GFX1250-NEXT: s_mov_b32 s1, s22
+; GFX1250-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX1250-NEXT: s_cmp_lg_u32 s35, 0
; GFX1250-NEXT: s_mov_b32 s3, s18
+; GFX1250-NEXT: s_add_co_ci_u32 s1, s1, s4
+; GFX1250-NEXT: s_cmp_lg_u32 s34, 0
+; GFX1250-NEXT: s_mov_b32 s4, s19
+; GFX1250-NEXT: s_add_co_ci_u32 s1, s1, s5
+; GFX1250-NEXT: s_cmp_lg_u32 s24, 0
; GFX1250-NEXT: s_mov_b32 s5, s20
+; GFX1250-NEXT: s_add_co_ci_u32 s1, s1, s6
+; GFX1250-NEXT: s_mov_b32 s6, s15
+; GFX1250-NEXT: s_add_co_i32 s7, s1, s7
+; GFX1250-NEXT: s_mov_b32 s1, s16
; GFX1250-NEXT: ; return to shader part epilog
;
; GFX13-LABEL: s_mul_i256:
; GFX13: ; %bb.0:
-; GFX13-NEXT: s_mov_b64 s[16:17], 0xffffffff
-; GFX13-NEXT: s_mov_b32 s23, 0
-; GFX13-NEXT: s_and_b64 s[30:31], s[8:9], s[16:17]
-; GFX13-NEXT: s_mov_b32 s20, s0
-; GFX13-NEXT: s_mov_b32 s21, s23
-; GFX13-NEXT: s_mov_b32 s22, s1
-; GFX13-NEXT: s_mul_u64 s[16:17], s[20:21], s[30:31]
-; GFX13-NEXT: s_mul_u64 s[18:19], s[22:23], s[30:31]
-; GFX13-NEXT: s_mov_b32 s24, s17
-; GFX13-NEXT: s_mov_b32 s25, s23
-; GFX13-NEXT: s_mov_b32 s26, s9
-; GFX13-NEXT: s_add_nc_u64 s[18:19], s[18:19], s[24:25]
-; GFX13-NEXT: s_mov_b32 s27, s23
-; GFX13-NEXT: s_mov_b32 s36, s2
-; GFX13-NEXT: s_mov_b32 s37, s23
-; GFX13-NEXT: s_mov_b32 s34, s19
-; GFX13-NEXT: s_mov_b32 s19, s23
-; GFX13-NEXT: s_mul_u64 s[24:25], s[20:21], s[26:27]
-; GFX13-NEXT: s_mov_b32 s28, s3
-; GFX13-NEXT: s_mul_u64 s[38:39], s[36:37], s[30:31]
-; GFX13-NEXT: s_mov_b32 s29, s23
-; GFX13-NEXT: s_add_nc_u64 s[24:25], s[24:25], s[18:19]
-; GFX13-NEXT: s_mul_u64 s[18:19], s[28:29], s[30:31]
-; GFX13-NEXT: s_mov_b32 s42, s39
-; GFX13-NEXT: s_mov_b32 s43, s23
-; GFX13-NEXT: s_mov_b32 s35, s23
-; GFX13-NEXT: s_mov_b32 s40, s25
-; GFX13-NEXT: s_mov_b32 s41, s23
-; GFX13-NEXT: s_add_nc_u64 s[18:19], s[18:19], s[42:43]
-; GFX13-NEXT: s_mul_u64 s[42:43], s[36:37], s[26:27]
-; GFX13-NEXT: s_add_nc_u64 s[34:35], s[34:35], s[40:41]
-; GFX13-NEXT: s_mov_b32 s40, s18
-; GFX13-NEXT: s_mov_b32 s46, s10
-; GFX13-NEXT: s_add_nc_u64 s[40:41], s[42:43], s[40:41]
-; GFX13-NEXT: s_mov_b32 s47, s23
-; GFX13-NEXT: s_mov_b32 s42, s23
-; GFX13-NEXT: s_mov_b32 s43, s40
-; GFX13-NEXT: s_mul_u64 s[48:49], s[20:21], s[46:47]
-; GFX13-NEXT: s_mov_b32 s39, s23
-; GFX13-NEXT: s_mul_u64 s[44:45], s[22:23], s[26:27]
-; GFX13-NEXT: s_or_b64 s[38:39], s[38:39], s[42:43]
-; GFX13-NEXT: s_mul_u64 s[42:43], s[22:23], s[46:47]
-; GFX13-NEXT: s_mov_b32 s50, s49
-; GFX13-NEXT: s_mov_b32 s51, s23
-; GFX13-NEXT: s_add_nc_u64 s[44:45], s[44:45], s[34:35]
-; GFX13-NEXT: s_mov_b32 s34, s11
-; GFX13-NEXT: s_mov_b32 s35, s23
-; GFX13-NEXT: s_add_nc_u64 s[42:43], s[42:43], s[50:51]
-; GFX13-NEXT: s_mul_u64 s[50:51], s[20:21], s[34:35]
-; GFX13-NEXT: s_mov_b32 s52, s42
-; GFX13-NEXT: s_mov_b32 s53, s23
-; GFX13-NEXT: s_add_co_u32 s17, s38, s44
-; GFX13-NEXT: s_add_co_ci_u32 s18, s39, s45
-; GFX13-NEXT: s_add_nc_u64 s[38:39], s[50:51], s[52:53]
-; GFX13-NEXT: s_mov_b32 s44, s23
-; GFX13-NEXT: s_mov_b32 s45, s38
-; GFX13-NEXT: s_mov_b32 s49, s23
-; GFX13-NEXT: s_cselect_b32 s25, -1, 0
-; GFX13-NEXT: s_or_b64 s[44:45], s[48:49], s[44:45]
-; GFX13-NEXT: s_mov_b32 s42, s43
-; GFX13-NEXT: s_mov_b32 s43, s23
-; GFX13-NEXT: s_mov_b32 s38, s39
-; GFX13-NEXT: s_mov_b32 s39, s23
-; GFX13-NEXT: s_mul_u64 s[48:49], s[22:23], s[34:35]
-; GFX13-NEXT: s_add_co_u32 s17, s44, s17
-; GFX13-NEXT: s_add_nc_u64 s[38:39], s[42:43], s[38:39]
-; GFX13-NEXT: s_add_co_ci_u32 s18, s45, s18
-; GFX13-NEXT: s_add_co_ci_u32 s33, s48, s38
-; GFX13-NEXT: s_add_co_ci_u32 s48, s49, s39
-; GFX13-NEXT: s_mov_b32 s38, s19
-; GFX13-NEXT: s_mov_b32 s39, s23
-; GFX13-NEXT: s_mov_b32 s40, s41
-; GFX13-NEXT: s_mov_b32 s41, s23
-; GFX13-NEXT: s_mul_u64 s[42:43], s[28:29], s[26:27]
-; GFX13-NEXT: s_add_nc_u64 s[38:39], s[38:39], s[40:41]
-; GFX13-NEXT: s_cmp_lg_u32 s25, 0
-; GFX13-NEXT: s_mul_u64 s[40:41], s[36:37], s[46:47]
-; GFX13-NEXT: s_add_co_ci_u32 s19, s42, s38
-; GFX13-NEXT: s_mul_u64 s[44:45], s[28:29], s[46:47]
-; GFX13-NEXT: s_add_co_ci_u32 s25, s43, s39
-; GFX13-NEXT: s_mov_b32 s38, s41
-; GFX13-NEXT: s_mov_b32 s39, s23
-; GFX13-NEXT: s_mul_u64 s[36:37], s[36:37], s[34:35]
-; GFX13-NEXT: s_add_nc_u64 s[38:39], s[44:45], s[38:39]
-; GFX13-NEXT: s_mov_b32 s43, s23
-; GFX13-NEXT: s_mov_b32 s42, s38
-; GFX13-NEXT: s_add_co_u32 s19, s19, s33
-; GFX13-NEXT: s_add_nc_u64 s[36:37], s[36:37], s[42:43]
-; GFX13-NEXT: s_mov_b32 s42, s23
-; GFX13-NEXT: s_mov_b32 s43, s36
-; GFX13-NEXT: s_mov_b32 s41, s23
-; GFX13-NEXT: s_add_co_ci_u32 s25, s25, s48
-; GFX13-NEXT: s_mov_b32 s44, s4
-; GFX13-NEXT: s_mov_b32 s45, s23
-; GFX13-NEXT: s_cselect_b32 s33, -1, 0
-; GFX13-NEXT: s_or_b64 s[40:41], s[40:41], s[42:43]
-; GFX13-NEXT: s_mul_u64 s[42:43], s[30:31], s[44:45]
-; GFX13-NEXT: s_add_co_u32 s19, s40, s19
-; GFX13-NEXT: s_mul_u64 s[44:45], s[26:27], s[44:45]
-; GFX13-NEXT: s_add_co_ci_u32 s25, s41, s25
-; GFX13-NEXT: s_mov_b32 s40, s43
-; GFX13-NEXT: s_mov_b32 s41, s23
-; GFX13-NEXT: s_mov_b32 s48, s12
-; GFX13-NEXT: s_add_nc_u64 s[40:41], s[44:45], s[40:41]
-; GFX13-NEXT: s_mov_b32 s44, s5
-; GFX13-NEXT: s_mov_b32 s45, s23
-; GFX13-NEXT: s_mov_b32 s49, s23
-; GFX13-NEXT: s_mov_b32 s46, s40
-; GFX13-NEXT: s_mul_u64 s[30:31], s[30:31], s[44:45]
-; GFX13-NEXT: s_mov_b32 s50, s13
-; GFX13-NEXT: s_mul_u64 s[52:53], s[48:49], s[20:21]
-; GFX13-NEXT: s_mov_b32 s51, s23
-; GFX13-NEXT: s_add_nc_u64 s[30:31], s[30:31], s[46:47]
-; GFX13-NEXT: s_mul_u64 s[20:21], s[50:51], s[20:21]
-; GFX13-NEXT: s_mov_b32 s46, s53
-; GFX13-NEXT: s_mov_b32 s54, s23
-; GFX13-NEXT: s_add_nc_u64 s[20:21], s[20:21], s[46:47]
-; GFX13-NEXT: s_mul_u64 s[46:47], s[48:49], s[22:23]
-; GFX13-NEXT: s_mov_b32 s48, s20
-; GFX13-NEXT: s_mov_b32 s55, s30
-; GFX13-NEXT: s_add_nc_u64 s[46:47], s[46:47], s[48:49]
-; GFX13-NEXT: s_mov_b32 s43, s23
-; GFX13-NEXT: s_mov_b32 s48, s23
-; GFX13-NEXT: s_mov_b32 s49, s46
-; GFX13-NEXT: s_mov_b32 s53, s23
-; GFX13-NEXT: s_cselect_b32 s56, -1, 0
-; GFX13-NEXT: s_or_b64 s[42:43], s[42:43], s[54:55]
-; GFX13-NEXT: s_or_b64 s[48:49], s[52:53], s[48:49]
-; GFX13-NEXT: s_mov_b32 s38, s39
-; GFX13-NEXT: s_add_co_u32 s20, s48, s42
-; GFX13-NEXT: s_add_co_ci_u32 s30, s49, s43
-; GFX13-NEXT: s_mov_b32 s39, s23
-; GFX13-NEXT: s_mov_b32 s36, s37
-; GFX13-NEXT: s_mov_b32 s37, s23
-; GFX13-NEXT: s_cselect_b32 s40, -1, 0
-; GFX13-NEXT: s_add_co_u32 s19, s19, s20
-; GFX13-NEXT: s_add_co_ci_u32 s20, s25, s30
-; GFX13-NEXT: s_mul_u64 s[28:29], s[28:29], s[34:35]
-; GFX13-NEXT: s_add_nc_u64 s[34:35], s[38:39], s[36:37]
-; GFX13-NEXT: s_cselect_b32 s25, -1, 0
-; GFX13-NEXT: s_and_b32 s30, s33, exec_lo
-; GFX13-NEXT: s_add_nc_u64 s[28:29], s[28:29], s[34:35]
-; GFX13-NEXT: s_mov_b32 s34, s41
-; GFX13-NEXT: s_mov_b32 s35, s23
-; GFX13-NEXT: s_mov_b32 s30, s31
-; GFX13-NEXT: s_mov_b32 s31, s23
-; GFX13-NEXT: s_mul_u64 s[4:5], s[4:5], s[10:11]
-; GFX13-NEXT: s_mul_u64 s[6:7], s[6:7], s[8:9]
-; GFX13-NEXT: s_mov_b32 s8, s21
-; GFX13-NEXT: s_mov_b32 s9, s23
-; GFX13-NEXT: s_mov_b32 s10, s47
-; GFX13-NEXT: s_mov_b32 s11, s23
-; GFX13-NEXT: s_cselect_b64 s[36:37], 1, 0
-; GFX13-NEXT: s_cmp_lg_u32 s56, 0
-; GFX13-NEXT: s_mul_u64 s[26:27], s[26:27], s[44:45]
-; GFX13-NEXT: s_add_nc_u64 s[30:31], s[34:35], s[30:31]
-; GFX13-NEXT: s_mul_u64 s[22:23], s[50:51], s[22:23]
-; GFX13-NEXT: s_add_nc_u64 s[4:5], s[6:7], s[4:5]
-; GFX13-NEXT: s_add_nc_u64 s[6:7], s[8:9], s[10:11]
-; GFX13-NEXT: s_mul_u64 s[0:1], s[0:1], s[14:15]
-; GFX13-NEXT: s_mul_u64 s[2:3], s[2:3], s[12:13]
-; GFX13-NEXT: s_add_co_ci_u32 s28, s28, s36
-; GFX13-NEXT: s_add_nc_u64 s[26:27], s[26:27], s[30:31]
-; GFX13-NEXT: s_add_nc_u64 s[6:7], s[22:23], s[6:7]
-; GFX13-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
-; GFX13-NEXT: s_add_co_ci_u32 s29, s29, s37
-; GFX13-NEXT: s_add_nc_u64 s[2:3], s[26:27], s[4:5]
-; GFX13-NEXT: s_add_nc_u64 s[0:1], s[6:7], s[0:1]
-; GFX13-NEXT: s_cmp_lg_u32 s40, 0
-; GFX13-NEXT: s_mov_b32 s4, s19
-; GFX13-NEXT: s_add_co_ci_u32 s0, s0, s2
-; GFX13-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX13-NEXT: s_mul_i32 s17, s0, s10
+; GFX13-NEXT: s_mul_i32 s19, s1, s9
+; GFX13-NEXT: s_mul_hi_u32 s18, s0, s10
+; GFX13-NEXT: s_mul_hi_u32 s20, s1, s9
+; GFX13-NEXT: s_add_co_u32 s17, s19, s17
+; GFX13-NEXT: s_add_co_ci_u32 s18, s20, s18
+; GFX13-NEXT: s_mul_i32 s20, s2, s8
+; GFX13-NEXT: s_mul_hi_u32 s21, s2, s8
+; GFX13-NEXT: s_cselect_b32 s19, 1, 0
+; GFX13-NEXT: s_add_co_u32 s17, s20, s17
+; GFX13-NEXT: s_mul_hi_u32 s16, s0, s8
+; GFX13-NEXT: s_add_co_ci_u32 s18, s21, s18
+; GFX13-NEXT: s_mul_i32 s21, s0, s9
+; GFX13-NEXT: s_mul_hi_u32 s22, s0, s9
+; GFX13-NEXT: s_cselect_b32 s20, 1, 0
+; GFX13-NEXT: s_add_co_u32 s16, s21, s16
+; GFX13-NEXT: s_add_co_ci_u32 s17, s22, s17
+; GFX13-NEXT: s_mul_i32 s22, s1, s8
+; GFX13-NEXT: s_mul_hi_u32 s23, s1, s8
+; GFX13-NEXT: s_cselect_b32 s21, 1, 0
+; GFX13-NEXT: s_add_co_u32 s16, s22, s16
+; GFX13-NEXT: s_add_co_ci_u32 s17, s23, s17
+; GFX13-NEXT: s_mul_i32 s23, s0, s12
+; GFX13-NEXT: s_mul_i32 s25, s1, s11
+; GFX13-NEXT: s_mul_hi_u32 s24, s0, s12
+; GFX13-NEXT: s_mul_hi_u32 s26, s1, s11
+; GFX13-NEXT: s_cselect_b32 s22, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s25, s23
+; GFX13-NEXT: s_add_co_ci_u32 s24, s26, s24
+; GFX13-NEXT: s_mul_i32 s26, s2, s10
+; GFX13-NEXT: s_mul_hi_u32 s27, s2, s10
+; GFX13-NEXT: s_cselect_b32 s25, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s26, s23
+; GFX13-NEXT: s_add_co_ci_u32 s24, s27, s24
+; GFX13-NEXT: s_mul_i32 s27, s3, s9
+; GFX13-NEXT: s_mul_hi_u32 s28, s3, s9
+; GFX13-NEXT: s_cselect_b32 s26, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s27, s23
+; GFX13-NEXT: s_add_co_ci_u32 s24, s28, s24
+; GFX13-NEXT: s_mul_i32 s28, s4, s8
+; GFX13-NEXT: s_mul_hi_u32 s29, s4, s8
+; GFX13-NEXT: s_cselect_b32 s27, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s28, s23
+; GFX13-NEXT: s_add_co_ci_u32 s24, s29, s24
+; GFX13-NEXT: s_mul_i32 s29, s0, s11
+; GFX13-NEXT: s_mul_hi_u32 s30, s0, s11
+; GFX13-NEXT: s_cselect_b32 s28, 1, 0
+; GFX13-NEXT: s_add_co_u32 s18, s29, s18
+; GFX13-NEXT: s_add_co_ci_u32 s23, s30, s23
+; GFX13-NEXT: s_mul_i32 s30, s1, s10
+; GFX13-NEXT: s_mul_hi_u32 s31, s1, s10
+; GFX13-NEXT: s_cselect_b32 s29, 1, 0
+; GFX13-NEXT: s_add_co_u32 s18, s30, s18
+; GFX13-NEXT: s_add_co_ci_u32 s23, s31, s23
+; GFX13-NEXT: s_mul_i32 s31, s2, s9
+; GFX13-NEXT: s_mul_hi_u32 s33, s2, s9
+; GFX13-NEXT: s_cselect_b32 s30, 1, 0
+; GFX13-NEXT: s_add_co_u32 s18, s31, s18
+; GFX13-NEXT: s_add_co_ci_u32 s23, s33, s23
+; GFX13-NEXT: s_mul_i32 s33, s3, s8
+; GFX13-NEXT: s_mul_hi_u32 s34, s3, s8
+; GFX13-NEXT: s_cselect_b32 s31, 1, 0
+; GFX13-NEXT: s_add_co_u32 s18, s33, s18
+; GFX13-NEXT: s_add_co_ci_u32 s23, s34, s23
+; GFX13-NEXT: s_cselect_b32 s33, 1, 0
+; GFX13-NEXT: s_cmp_lg_u32 s21, 0
+; GFX13-NEXT: s_mul_hi_u32 s34, s1, s13
+; GFX13-NEXT: s_cselect_b32 s21, 1, 0
+; GFX13-NEXT: s_cmp_lg_u32 s22, 0
+; GFX13-NEXT: s_mul_hi_u32 s22, s0, s14
+; GFX13-NEXT: s_add_co_ci_u32 s18, s21, s18
+; GFX13-NEXT: s_cselect_b32 s21, 1, 0
+; GFX13-NEXT: s_cmp_lg_u32 s19, 0
+; GFX13-NEXT: s_mul_hi_u32 s35, s1, s12
+; GFX13-NEXT: s_cselect_b32 s19, 1, 0
+; GFX13-NEXT: s_cmp_lg_u32 s20, 0
+; GFX13-NEXT: s_mul_hi_u32 s36, s2, s11
+; GFX13-NEXT: s_add_co_ci_u32 s19, s19, 0
+; GFX13-NEXT: s_cmp_lg_u32 s21, 0
+; GFX13-NEXT: s_mul_i32 s21, s0, s14
+; GFX13-NEXT: s_add_co_ci_u32 s19, s19, s23
+; GFX13-NEXT: s_mul_i32 s23, s1, s13
+; GFX13-NEXT: s_cselect_b32 s20, 1, 0
+; GFX13-NEXT: s_add_co_u32 s21, s23, s21
+; GFX13-NEXT: s_mul_i32 s23, s2, s12
+; GFX13-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX13-NEXT: s_mul_hi_u32 s34, s2, s12
+; GFX13-NEXT: s_add_co_u32 s21, s23, s21
+; GFX13-NEXT: s_mul_i32 s23, s3, s11
+; GFX13-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX13-NEXT: s_mul_hi_u32 s34, s3, s11
+; GFX13-NEXT: s_add_co_u32 s21, s23, s21
+; GFX13-NEXT: s_mul_i32 s23, s4, s10
+; GFX13-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX13-NEXT: s_mul_hi_u32 s34, s4, s10
+; GFX13-NEXT: s_add_co_u32 s21, s23, s21
+; GFX13-NEXT: s_mul_i32 s23, s5, s9
+; GFX13-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX13-NEXT: s_mul_hi_u32 s34, s5, s9
+; GFX13-NEXT: s_add_co_u32 s21, s23, s21
+; GFX13-NEXT: s_mul_i32 s23, s6, s8
+; GFX13-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX13-NEXT: s_mul_hi_u32 s34, s6, s8
+; GFX13-NEXT: s_add_co_u32 s21, s23, s21
+; GFX13-NEXT: s_mul_i32 s23, s0, s13
+; GFX13-NEXT: s_add_co_ci_u32 s22, s34, s22
+; GFX13-NEXT: s_mul_hi_u32 s34, s0, s13
+; GFX13-NEXT: s_add_co_u32 s23, s23, s24
+; GFX13-NEXT: s_add_co_ci_u32 s21, s34, s21
+; GFX13-NEXT: s_mul_i32 s34, s1, s12
+; GFX13-NEXT: s_cselect_b32 s24, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s34, s23
+; GFX13-NEXT: s_add_co_ci_u32 s21, s35, s21
+; GFX13-NEXT: s_mul_i32 s35, s2, s11
+; GFX13-NEXT: s_cselect_b32 s34, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s35, s23
+; GFX13-NEXT: s_add_co_ci_u32 s21, s36, s21
+; GFX13-NEXT: s_mul_i32 s36, s3, s10
+; GFX13-NEXT: s_mul_hi_u32 s37, s3, s10
+; GFX13-NEXT: s_cselect_b32 s35, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s36, s23
+; GFX13-NEXT: s_add_co_ci_u32 s21, s37, s21
+; GFX13-NEXT: s_mul_i32 s37, s4, s9
+; GFX13-NEXT: s_mul_hi_u32 s38, s4, s9
+; GFX13-NEXT: s_cselect_b32 s36, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s37, s23
+; GFX13-NEXT: s_add_co_ci_u32 s21, s38, s21
+; GFX13-NEXT: s_mul_i32 s38, s5, s8
+; GFX13-NEXT: s_mul_hi_u32 s39, s5, s8
+; GFX13-NEXT: s_cselect_b32 s37, 1, 0
+; GFX13-NEXT: s_add_co_u32 s23, s38, s23
+; GFX13-NEXT: s_add_co_ci_u32 s21, s39, s21
+; GFX13-NEXT: s_cselect_b32 s38, 1, 0
+; GFX13-NEXT: s_cmp_lg_u32 s29, 0
+; GFX13-NEXT: s_mul_i32 s1, s1, s14
+; GFX13-NEXT: s_cselect_b32 s29, 1, 0
+; GFX13-NEXT: s_cmp_lg_u32 s30, 0
+; GFX13-NEXT: s_mul_i32 s2, s2, s13
+; GFX13-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX13-NEXT: s_cmp_lg_u32 s31, 0
+; GFX13-NEXT: s_mul_i32 s3, s3, s12
+; GFX13-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX13-NEXT: s_cmp_lg_u32 s33, 0
+; GFX13-NEXT: s_mul_i32 s4, s4, s11
+; GFX13-NEXT: s_add_co_ci_u32 s29, s29, 0
+; GFX13-NEXT: s_cmp_lg_u32 s20, 0
+; GFX13-NEXT: s_mul_i32 s5, s5, s10
+; GFX13-NEXT: s_add_co_ci_u32 s20, s29, s23
+; GFX13-NEXT: s_cselect_b32 s23, 1, 0
; GFX13-NEXT: s_cmp_lg_u32 s25, 0
+; GFX13-NEXT: s_mul_i32 s6, s6, s9
+; GFX13-NEXT: s_cselect_b32 s25, 1, 0
+; GFX13-NEXT: s_cmp_lg_u32 s26, 0
+; GFX13-NEXT: s_mul_i32 s26, s0, s15
+; GFX13-NEXT: s_add_co_ci_u32 s25, s25, 0
+; GFX13-NEXT: s_cmp_lg_u32 s27, 0
+; GFX13-NEXT: s_mul_i32 s7, s7, s8
+; GFX13-NEXT: s_add_co_ci_u32 s25, s25, 0
+; GFX13-NEXT: s_cmp_lg_u32 s28, 0
+; GFX13-NEXT: s_mul_i32 s0, s0, s8
+; GFX13-NEXT: s_add_co_ci_u32 s25, s25, 0
+; GFX13-NEXT: s_cmp_lg_u32 s23, 0
+; GFX13-NEXT: s_add_co_ci_u32 s15, s25, s21
+; GFX13-NEXT: s_add_co_ci_u32 s21, s22, s26
+; GFX13-NEXT: s_cmp_lg_u32 s38, 0
+; GFX13-NEXT: s_add_co_ci_u32 s1, s21, s1
+; GFX13-NEXT: s_cmp_lg_u32 s37, 0
+; GFX13-NEXT: s_add_co_ci_u32 s1, s1, s2
+; GFX13-NEXT: s_cmp_lg_u32 s36, 0
; GFX13-NEXT: s_mov_b32 s2, s17
-; GFX13-NEXT: s_add_co_ci_u32 s6, s28, s0
-; GFX13-NEXT: s_add_co_ci_u32 s7, s29, s1
-; GFX13-NEXT: s_mov_b32 s0, s16
-; GFX13-NEXT: s_mov_b32 s1, s24
+; GFX13-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX13-NEXT: s_cmp_lg_u32 s35, 0
; GFX13-NEXT: s_mov_b32 s3, s18
+; GFX13-NEXT: s_add_co_ci_u32 s1, s1, s4
+; GFX13-NEXT: s_cmp_lg_u32 s34, 0
+; GFX13-NEXT: s_mov_b32 s4, s19
+; GFX13-NEXT: s_add_co_ci_u32 s1, s1, s5
+; GFX13-NEXT: s_cmp_lg_u32 s24, 0
; GFX13-NEXT: s_mov_b32 s5, s20
+; GFX13-NEXT: s_add_co_ci_u32 s1, s1, s6
+; GFX13-NEXT: s_mov_b32 s6, s15
+; GFX13-NEXT: s_add_co_i32 s7, s1, s7
+; GFX13-NEXT: s_mov_b32 s1, s16
; GFX13-NEXT: ; return to shader part epilog
%result = mul i256 %num, %den
%cast = bitcast i256 %result to <8 x i32>
@@ -3431,11 +3728,15 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: v_mov_b32_e32 v0, 0x50
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s2, s[2:3], 0x0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v0, 0
+; GFX7-NEXT: s_load_dword s3, s[2:3], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mul_hi_u32 v0, s3, v0
+; GFX7-NEXT: s_mul_i32 s4, s3, 0x50
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_readfirstlane_b32 s5, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -3445,10 +3746,14 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
; GFX8-NEXT: v_mov_b32_e32 v0, 0x50
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, 0
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: s_mulk_i32 s2, 0x50
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -3457,10 +3762,10 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT: s_load_dword s3, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_hi_u32 s3, s2, 0x50
-; GFX9-NEXT: s_mulk_i32 s2, 0x50
+; GFX9-NEXT: s_mul_i32 s2, s3, 0x50
+; GFX9-NEXT: s_mul_hi_u32 s3, s3, 0x50
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -3471,25 +3776,25 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX10-NEXT: s_load_dword s3, s[2:3], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mul_i32 s3, s2, 0x50
-; GFX10-NEXT: s_mul_hi_u32 s2, s2, 0x50
-; GFX10-NEXT: v_mov_b32_e32 v0, s3
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_mul_i32 s2, s3, 0x50
+; GFX10-NEXT: s_mul_hi_u32 s3, s3, 0x50
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_mul_u64_zext_with_sregs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-NEXT: s_load_b32 s3, s[2:3], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mul_i32 s3, s2, 0x50
-; GFX11-NEXT: s_mul_hi_u32 s2, s2, 0x50
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s3
+; GFX11-NEXT: s_mul_i32 s2, s3, 0x50
+; GFX11-NEXT: s_mul_hi_u32 s3, s3, 0x50
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
@@ -3606,28 +3911,26 @@ define amdgpu_ps void @s_mul_u64_sext_with_vregs(ptr addrspace(1) %out, ptr addr
;
; GFX12-LABEL: s_mul_u64_sext_with_vregs:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v2, v[2:3], off
+; GFX12-NEXT: global_load_b32 v4, v[2:3], off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mad_co_i64_i32 v[2:3], null, 0x50, v2, 0
+; GFX12-NEXT: v_mad_co_i64_i32 v[2:3], null, 0x50, v4, 0
; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-NEXT: s_endpgm
;
; GFX1250-LABEL: s_mul_u64_sext_with_vregs:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_load_b32 v2, v[2:3], off
+; GFX1250-NEXT: global_load_b32 v4, v[2:3], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[2:3], 0x50, v[2:3]
+; GFX1250-NEXT: v_mad_nc_i64_i32 v[2:3], 0x50, v4, 0
; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1250-NEXT: s_endpgm
;
; GFX13-LABEL: s_mul_u64_sext_with_vregs:
; GFX13: ; %bb.0:
-; GFX13-NEXT: global_load_b32 v2, v[2:3], off
+; GFX13-NEXT: global_load_b32 v4, v[2:3], off
; GFX13-NEXT: s_wait_loadcnt 0x0
-; GFX13-NEXT: v_mad_co_i64_i32 v[2:3], null, 0x50, v2, 0
+; GFX13-NEXT: v_mad_co_i64_i32 v[2:3], null, 0x50, v4, 0
; GFX13-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX13-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
@@ -3643,11 +3946,18 @@ define amdgpu_kernel void @s_mul_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: v_mov_b32_e32 v0, 0x50
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s2, s[2:3], 0x0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_i64_i32 v[0:1], s[4:5], s2, v0, 0
+; GFX7-NEXT: s_load_dword s3, s[2:3], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mul_hi_u32 v0, s3, v0
+; GFX7-NEXT: s_ashr_i32 s5, s3, 31
+; GFX7-NEXT: s_mul_i32 s4, s3, 0x50
+; GFX7-NEXT: s_mulk_i32 s5, 0x50
+; GFX7-NEXT: v_readfirstlane_b32 s3, v0
+; GFX7-NEXT: s_add_u32 s5, s5, s3
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -3657,10 +3967,17 @@ define amdgpu_kernel void @s_mul_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX8-NEXT: v_mov_b32_e32 v0, 0x50
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_i64_i32 v[0:1], s[2:3], s2, v0, 0
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: s_ashr_i32 s3, s2, 31
+; GFX8-NEXT: s_mulk_i32 s3, 0x50
+; GFX8-NEXT: s_mulk_i32 s2, 0x50
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_add_u32 s3, s3, s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -3669,10 +3986,13 @@ define amdgpu_kernel void @s_mul_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-NEXT: s_load_dword s3, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_hi_i32 s3, s2, 0x50
-; GFX9-NEXT: s_mulk_i32 s2, 0x50
+; GFX9-NEXT: s_ashr_i32 s4, s3, 31
+; GFX9-NEXT: s_mul_i32 s2, s3, 0x50
+; GFX9-NEXT: s_mul_hi_u32 s3, s3, 0x50
+; GFX9-NEXT: s_mulk_i32 s4, 0x50
+; GFX9-NEXT: s_add_u32 s3, s4, s3
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -3714,14 +4034,14 @@ define amdgpu_kernel void @s_mul_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX12-LABEL: s_mul_u64_sext_with_sregs:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_mul_u64 s[2:3], s[2:3], 0x50
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
;
@@ -3743,14 +4063,14 @@ define amdgpu_kernel void @s_mul_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX13-LABEL: s_mul_u64_sext_with_sregs:
; GFX13: ; %bb.0:
; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
; GFX13-NEXT: s_wait_kmcnt 0x0
; GFX13-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX13-NEXT: s_wait_kmcnt 0x0
; GFX13-NEXT: s_ashr_i32 s3, s2, 31
; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13-NEXT: s_mul_u64 s[2:3], s[2:3], 0x50
-; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
-; GFX13-NEXT: v_mov_b32_e32 v1, s3
+; GFX13-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX13-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
index 10877872b4ef6..f6d518c9840b5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
define amdgpu_ps i16 @s_or_i16(i16 inreg %num, i16 inreg %den) {
; GCN-LABEL: s_or_i16:
@@ -688,8 +688,7 @@ define amdgpu_ps void @s_or_u64_sext_with_vregs(ptr addrspace(1) %out, ptr addrs
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: buffer_load_dword v2, v[2:3], s[0:3], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_ashrrev_i32_e32 v3, 31, v2
@@ -754,11 +753,11 @@ define amdgpu_kernel void @s_or_u64_sext_with_sregs(ptr addrspace(1) %out, ptr a
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_ashr_i32 s5, s4, 31
-; GFX7-NEXT: s_or_b32 s4, s4, 0x50
+; GFX7-NEXT: s_or_b64 s[4:5], s[4:5], 0x50
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -768,15 +767,15 @@ define amdgpu_kernel void @s_or_u64_sext_with_sregs(ptr addrspace(1) %out, ptr a
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_load_dword s2, s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_ashr_i32 s0, s2, 31
-; GFX8-NEXT: s_or_b32 s1, s2, 0x50
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, s0
-; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT: s_ashr_i32 s3, s2, 31
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], 0x50
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: s_or_u64_sext_with_sregs:
@@ -787,7 +786,7 @@ define amdgpu_kernel void @s_or_u64_sext_with_sregs(ptr addrspace(1) %out, ptr a
; GFX9-NEXT: s_load_dword s2, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_ashr_i32 s3, s2, 31
-; GFX9-NEXT: s_or_b32 s2, s2, 0x50
+; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], 0x50
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -800,37 +799,37 @@ define amdgpu_kernel void @s_or_u64_sext_with_sregs(ptr addrspace(1) %out, ptr a
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s2, s[2:3], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_or_b32 s3, s2, 0x50
-; GFX10-NEXT: s_ashr_i32 s2, s2, 31
-; GFX10-NEXT: v_mov_b32_e32 v0, s3
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_ashr_i32 s3, s2, 31
+; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], 0x50
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_or_u64_sext_with_sregs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_or_b32 s3, s2, 0x50
-; GFX11-NEXT: s_ashr_i32 s2, s2, 31
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s3
+; GFX11-NEXT: s_ashr_i32 s3, s2, 31
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], 0x50
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_or_u64_sext_with_sregs:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_or_b32 s3, s2, 0x50
-; GFX12-NEXT: s_ashr_i32 s2, s2, 31
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, s3
+; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_or_b64 s[2:3], s[2:3], 0x50
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
index b030ff1455cec..9415318d40c00 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; FIXME: regbankcombiner regression, related to:
@@ -246,15 +246,19 @@ define i64 @v_orn2_i64(i64 %src0, i64 %src1) {
; GCN-LABEL: v_orn2_i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_bfi_b32 v1, v3, v1, -1
-; GCN-NEXT: v_bfi_b32 v0, v2, v0, -1
+; GCN-NEXT: v_xor_b32_e32 v2, -1, v2
+; GCN-NEXT: v_xor_b32_e32 v3, -1, v3
+; GCN-NEXT: v_or_b32_e32 v0, v0, v2
+; GCN-NEXT: v_or_b32_e32 v1, v1, v3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_orn2_i64:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_bfi_b32 v0, v2, v0, -1
-; GFX10PLUS-NEXT: v_bfi_b32 v1, v3, v1, -1
+; GFX10PLUS-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX10PLUS-NEXT: v_xor_b32_e32 v3, -1, v3
+; GFX10PLUS-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10PLUS-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%not.src1 = xor i64 %src1, -1
%or = or i64 %src0, %not.src1
@@ -264,14 +268,18 @@ define i64 @v_orn2_i64(i64 %src0, i64 %src1) {
define amdgpu_ps <2 x float> @v_orn2_i64_sv(i64 inreg %src0, i64 %src1) {
; GCN-LABEL: v_orn2_i64_sv:
; GCN: ; %bb.0:
-; GCN-NEXT: v_bfi_b32 v1, v1, s3, -1
-; GCN-NEXT: v_bfi_b32 v0, v0, s2, -1
+; GCN-NEXT: v_xor_b32_e32 v0, -1, v0
+; GCN-NEXT: v_xor_b32_e32 v1, -1, v1
+; GCN-NEXT: v_or_b32_e32 v0, s2, v0
+; GCN-NEXT: v_or_b32_e32 v1, s3, v1
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: v_orn2_i64_sv:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_bfi_b32 v0, v0, s2, -1
-; GFX10PLUS-NEXT: v_bfi_b32 v1, v1, s3, -1
+; GFX10PLUS-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX10PLUS-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX10PLUS-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX10PLUS-NEXT: v_or_b32_e32 v1, s3, v1
; GFX10PLUS-NEXT: ; return to shader part epilog
%not.src1 = xor i64 %src1, -1
%or = or i64 %src0, %not.src1
@@ -282,15 +290,29 @@ define amdgpu_ps <2 x float> @v_orn2_i64_sv(i64 inreg %src0, i64 %src1) {
define amdgpu_ps <2 x float> @v_orn2_i64_vs(i64 %src0, i64 inreg %src1) {
; GCN-LABEL: v_orn2_i64_vs:
; GCN: ; %bb.0:
-; GCN-NEXT: v_bfi_b32 v1, s3, v1, -1
-; GCN-NEXT: v_bfi_b32 v0, s2, v0, -1
+; GCN-NEXT: s_not_b64 s[0:1], s[2:3]
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_or_b32_e32 v0, v0, v2
+; GCN-NEXT: v_or_b32_e32 v1, v1, v3
; GCN-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: v_orn2_i64_vs:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_bfi_b32 v0, s2, v0, -1
-; GFX10PLUS-NEXT: v_bfi_b32 v1, s3, v1, -1
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: v_orn2_i64_vs:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_not_b64 s[0:1], s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_orn2_i64_vs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_not_b64 s[0:1], s[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: ; return to shader part epilog
%not.src1 = xor i64 %src1, -1
%or = or i64 %src0, %not.src1
%cast = bitcast i64 %or to <2 x float>
@@ -704,16 +726,25 @@ define amdgpu_ps i48 @s_orn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1)
; GFX10-NEXT: s_or_b32 s1, s3, s2
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_orn2_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-NEXT: s_or_not1_b32 s0, s1, s0
-; GFX11-NEXT: s_or_not1_b32 s1, s2, s4
-; GFX11-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_or_b32 s1, s3, s2
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_orn2_v3i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-TRUE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-TRUE16-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_orn2_v3i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s2, 16
+; GFX11-FAKE16-NEXT: s_or_not1_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s2, s4
+; GFX11-FAKE16-NEXT: s_xor_b32 s2, s5, 0xffff
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s3, s2
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%or = or <3 x i16> %src0, %not.src1
%cast = bitcast <3 x i16> %or to i48
@@ -750,16 +781,25 @@ define amdgpu_ps i48 @s_orn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inre
; GFX10-NEXT: s_or_b32 s1, s2, s3
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_orn2_v3i16_commute:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-NEXT: s_or_not1_b32 s0, s1, s0
-; GFX11-NEXT: s_or_not1_b32 s1, s2, s4
-; GFX11-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_or_b32 s1, s2, s3
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_orn2_v3i16_commute:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-TRUE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-TRUE16-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_orn2_v3i16_commute:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s2, 16
+; GFX11-FAKE16-NEXT: s_or_not1_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s2, s4
+; GFX11-FAKE16-NEXT: s_xor_b32 s2, s5, 0xffff
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s2, s3
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%or = or <3 x i16> %not.src1, %src0
%cast = bitcast <3 x i16> %or to i48
@@ -819,21 +859,33 @@ define amdgpu_ps { i48, i48 } @s_orn2_v3i16_multi_use(<3 x i16> inreg %src0, <3
; GFX10-NEXT: s_mov_b32 s3, s4
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_orn2_v3i16_multi_use:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-NEXT: s_not_b32 s6, s4
-; GFX11-NEXT: s_not_b32 s1, s0
-; GFX11-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX11-NEXT: s_or_not1_b32 s0, s7, s0
-; GFX11-NEXT: s_or_not1_b32 s1, s2, s4
-; GFX11-NEXT: s_not_b32 s4, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_or_not1_b32 s1, s3, s5
-; GFX11-NEXT: s_mov_b32 s2, s6
-; GFX11-NEXT: s_mov_b32 s3, s4
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_orn2_v3i16_multi_use:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-TRUE16-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX11-TRUE16-NEXT: s_or_b64 s[0:1], s[2:3], s[4:5]
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s4, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_orn2_v3i16_multi_use:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s4
+; GFX11-FAKE16-NEXT: s_not_b32 s1, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s2, 16
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s1
+; GFX11-FAKE16-NEXT: s_or_not1_b32 s0, s7, s0
+; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s2, s4
+; GFX11-FAKE16-NEXT: s_not_b32 s4, s5
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s3, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, s4
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%or = or <3 x i16> %src0, %not.src1
%cast.0 = bitcast <3 x i16> %or to i48
@@ -856,38 +908,20 @@ define <3 x i16> @v_orn2_v3i16(<3 x i16> %src0, <3 x i16> %src1) {
; GFX9-LABEL: v_orn2_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v3, -11, v3
; GFX9-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_xor_b32_e32 v3, -11, v3
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_orn2_v3i16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX10-NEXT: v_xor_b32_e32 v3, -11, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_orn2_v3i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v3.l, -11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_orn2_v3i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, -1, v2
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v3, -11, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX10PLUS-LABEL: v_orn2_v3i16:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10PLUS-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX10PLUS-NEXT: v_xor_b32_e32 v3, -11, v3
+; GFX10PLUS-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10PLUS-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -11>
%or = or <3 x i16> %src0, %not.src1
ret <3 x i16> %or
@@ -1097,17 +1131,35 @@ define <4 x i16> @v_orn2_v4i16(<4 x i16> %src0, <4 x i16> %src1) {
; GFX6-LABEL: v_orn2_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfi_b32 v0, v2, v0, -1
-; GFX6-NEXT: v_bfi_b32 v1, v3, v1, -1
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT: v_xor_b32_e32 v3, -1, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_orn2_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT: v_xor_b32_e32 v3, -1, v3
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v2, -1, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_orn2_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir
index dc73f1061dac8..243b15ad3d646 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-abs.mir
@@ -14,8 +14,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ABS:%[0-9]+]]:sgpr(s32) = G_ABS [[SEXT]]
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:sgpr(i32) = G_ABS [[SEXT]]
%1:_(s32) = COPY $sgpr0
%2:_(s16) = G_TRUNC %1
%5:_(s16) = G_ABS %2
@@ -54,13 +54,13 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[BITCAST]], 16
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[ABS:%[0-9]+]]:sgpr(s32) = G_ABS [[SEXT_INREG]]
- ; CHECK-NEXT: [[ABS1:%[0-9]+]]:sgpr(s32) = G_ABS [[ASHR]]
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ABS]](s32), [[ABS1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(i32) = G_SEXT_INREG [[BITCAST]], 16
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(i32) = G_ASHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:sgpr(i32) = G_ABS [[SEXT_INREG]]
+ ; CHECK-NEXT: [[ABS1:%[0-9]+]]:sgpr(i32) = G_ABS [[ASHR]]
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ABS]](i32), [[ABS1]](i32)
%1:_(<2 x s16>) = COPY $sgpr0
%5:_(<2 x s16>) = G_ABS %1
...
@@ -77,8 +77,8 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s16) = G_CONSTANT i16 0
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s16>) = G_BUILD_VECTOR [[C]](i16), [[C]](i16)
; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(<2 x s16>) = G_SUB [[BUILD_VECTOR]], [[COPY]]
; CHECK-NEXT: [[SMAX:%[0-9]+]]:vgpr(<2 x s16>) = G_SMAX [[COPY]], [[SUB]]
%1:_(<2 x s16>) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.mir
index 097372a957461..51f0a84b2669a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.mir
@@ -14,10 +14,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[ANYEXT]], [[ANYEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ADD]](s32)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[ANYEXT]], [[ANYEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ADD]](i32)
; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s16) = G_AND [[TRUNC2]], [[TRUNC2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -291,11 +291,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[UADDO:%[0-9]+]]:sgpr(s32), [[UADDO1:%[0-9]+]]:sgpr(s32) = G_UADDO [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[UADDO1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C]], [[C1]]
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:sgpr(s32), [[UADDO1:%[0-9]+]]:sgpr(i32) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[UADDO1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[SELECT]], [[UADDO]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -391,12 +392,13 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(s32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[UADDE1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](s32), [[C]], [[C1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(i32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[UADDE1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](i32), [[C1]], [[C2]]
; CHECK-NEXT: [[AND2:%[0-9]+]]:sgpr(s32) = G_AND [[UADDE]], [[SELECT]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -507,12 +509,13 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(s32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[UADDE1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](s32), [[C]], [[C1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(i32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[UADDE1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](i32), [[C1]], [[C2]]
; CHECK-NEXT: [[AND2:%[0-9]+]]:sgpr(s32) = G_AND [[UADDE]], [[SELECT]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir
index 30c958fcb192a..dffd212f03e15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s16.mir
@@ -14,10 +14,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[ANYEXT]], [[ANYEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ADD]](s32)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[ANYEXT]], [[ANYEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ADD]](i32)
; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s16) = G_AND [[TRUNC2]], [[TRUNC2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir
index 01eb39111b0ab..89a110b7414b8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.v2s16.mir
@@ -13,14 +13,14 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[BITCAST]], [[BITCAST1]]
- ; CHECK-NEXT: [[ADD1:%[0-9]+]]:sgpr(s32) = G_ADD [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ADD]](s32), [[ADD1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[BITCAST]], [[BITCAST1]]
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:sgpr(i32) = G_ADD [[LSHR]], [[LSHR1]]
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ADD]](i32), [[ADD1]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s16) = G_CONSTANT i16 255
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(<2 x s16>) = G_AND [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir
index 999b0750e6068..ad37e35fe47bd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir
@@ -34,9 +34,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[COPY]](<4 x s32>), [[C1]](s32), [[COPY1]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[COPY]](<4 x s32>), [[C1]](i32), [[COPY1]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
%0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
%1:_(s32) = COPY $vgpr0
%2:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD %0, %1, 0 :: (dereferenceable invariant load (s128))
@@ -57,9 +58,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(i32) = COPY [[COPY1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; CHECK-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; CHECK-NEXT: {{ $}}
@@ -72,18 +73,18 @@ body: |
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .2:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C1]](s32), [[COPY2]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
+ ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C1]](i32), [[COPY2]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
; CHECK-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; CHECK-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; CHECK-NEXT: {{ $}}
@@ -93,6 +94,7 @@ body: |
; CHECK-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .4:
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
%0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(s32) = COPY $sgpr0
%2:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD %0, %1, 0 :: (dereferenceable invariant load (s128))
@@ -113,8 +115,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; CHECK-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; CHECK-NEXT: {{ $}}
@@ -127,18 +129,18 @@ body: |
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .2:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C1]](s32), [[COPY1]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
+ ; CHECK-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C1]](i32), [[COPY1]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
; CHECK-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; CHECK-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; CHECK-NEXT: {{ $}}
@@ -148,6 +150,7 @@ body: |
; CHECK-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .4:
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[AMDGPU_BUFFER_LOAD]](<4 x s32>)
%0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(s32) = COPY $vgpr4
%2:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD %0, %1, 0 :: (dereferenceable invariant load (s128))
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir
index 97410906e0818..59df99d24d38e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.div.fmas.mir
@@ -16,11 +16,11 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fmas), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[AMDGPU_COPY_VCC_SCC]](s1)
; CHECK-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[INT]]
%0:_(s32) = COPY $sgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
index 3ecdf7788a546..33a80815e5c3b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
@@ -126,17 +126,17 @@ define amdgpu_ps void @load_1d_vgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 %s) {
; FAST-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; FAST-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; FAST-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
- ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
+ ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; FAST-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; FAST-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; FAST-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
- ; FAST-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
- ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; FAST-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
+ ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; FAST-NEXT: {{ $}}
; FAST-NEXT: bb.3:
; FAST-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -186,17 +186,17 @@ define amdgpu_ps void @load_1d_vgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 %s) {
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; GREEDY-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
- ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
+ ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; GREEDY-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; GREEDY-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; GREEDY-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
- ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
- ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
+ ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GREEDY-NEXT: {{ $}}
; GREEDY-NEXT: bb.3:
; GREEDY-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -253,17 +253,17 @@ define amdgpu_ps void @load_1d_sgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 inreg
; FAST-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; FAST-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; FAST-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
- ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
+ ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; FAST-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; FAST-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; FAST-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
- ; FAST-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
- ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; FAST-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
+ ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; FAST-NEXT: {{ $}}
; FAST-NEXT: bb.3:
; FAST-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -314,17 +314,17 @@ define amdgpu_ps void @load_1d_sgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 inreg
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; GREEDY-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
- ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<8 x i32>)
+ ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; GREEDY-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; GREEDY-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; GREEDY-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
- ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
- ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
+ ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GREEDY-NEXT: {{ $}}
; GREEDY-NEXT: bb.3:
; GREEDY-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
index 34faa822d937a..2bab907079634 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
@@ -151,17 +151,17 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp(<8 x i32> %rsr
; FAST-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; FAST-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; FAST-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
- ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
+ ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; FAST-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; FAST-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; FAST-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
- ; FAST-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
- ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; FAST-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
+ ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; FAST-NEXT: {{ $}}
; FAST-NEXT: bb.3:
; FAST-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -216,17 +216,17 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp(<8 x i32> %rsr
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; GREEDY-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
- ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
+ ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; GREEDY-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; GREEDY-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; GREEDY-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
- ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
- ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND2]](s1)
+ ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT8]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GREEDY-NEXT: {{ $}}
; GREEDY-NEXT: bb.3:
; GREEDY-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -283,13 +283,13 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp(<8 x i32> inre
; FAST-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; FAST-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; FAST-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; FAST-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; FAST-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<4 x i32>)
- ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; FAST-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; FAST-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<4 x i32>)
+ ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; FAST-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; FAST-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; FAST-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; FAST-NEXT: {{ $}}
; FAST-NEXT: bb.3:
; FAST-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -340,13 +340,13 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp(<8 x i32> inre
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GREEDY-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GREEDY-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GREEDY-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<4 x i32>)
- ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GREEDY-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GREEDY-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<4 x i32>)
+ ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GREEDY-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GREEDY-NEXT: {{ $}}
; GREEDY-NEXT: bb.3:
; GREEDY-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -407,14 +407,14 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsr
; FAST-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; FAST-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; FAST-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
- ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; FAST-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; FAST-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
+ ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; FAST-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; FAST-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; FAST-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; FAST-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; FAST-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; FAST-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
; FAST-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32), [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
; FAST-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV16]](i32)
@@ -422,14 +422,14 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsr
; FAST-NEXT: [[INTRINSIC_CONVERGENT10:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV18]](i32)
; FAST-NEXT: [[INTRINSIC_CONVERGENT11:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV19]](i32)
; FAST-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT8]](i32), [[INTRINSIC_CONVERGENT9]](i32), [[INTRINSIC_CONVERGENT10]](i32), [[INTRINSIC_CONVERGENT11]](i32)
- ; FAST-NEXT: [[UV20:%[0-9]+]]:vgpr(s64), [[UV21:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; FAST-NEXT: [[UV22:%[0-9]+]]:sgpr(s64), [[UV23:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<4 x i32>)
- ; FAST-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV22]](s64), [[UV20]]
+ ; FAST-NEXT: [[UV20:%[0-9]+]]:vgpr(i64), [[UV21:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; FAST-NEXT: [[UV22:%[0-9]+]]:sgpr(i64), [[UV23:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<4 x i32>)
+ ; FAST-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV22]](i64), [[UV20]]
; FAST-NEXT: [[AND3:%[0-9]+]]:vcc(s1) = G_AND [[AND2]], [[ICMP4]]
- ; FAST-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV23]](s64), [[UV21]]
+ ; FAST-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV23]](i64), [[UV21]]
; FAST-NEXT: [[AND4:%[0-9]+]]:vcc(s1) = G_AND [[AND3]], [[ICMP5]]
- ; FAST-NEXT: [[INTRINSIC_CONVERGENT12:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND4]](s1)
- ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; FAST-NEXT: [[INTRINSIC_CONVERGENT12:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND4]](s1)
+ ; FAST-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; FAST-NEXT: {{ $}}
; FAST-NEXT: bb.3:
; FAST-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -484,14 +484,14 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsr
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT6:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV6]](i32)
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT7:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV7]](i32)
; GREEDY-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32), [[INTRINSIC_CONVERGENT4]](i32), [[INTRINSIC_CONVERGENT5]](i32), [[INTRINSIC_CONVERGENT6]](i32), [[INTRINSIC_CONVERGENT7]](i32)
- ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(s64), [[UV9:%[0-9]+]]:vgpr(s64), [[UV10:%[0-9]+]]:vgpr(s64), [[UV11:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
- ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(s64), [[UV13:%[0-9]+]]:sgpr(s64), [[UV14:%[0-9]+]]:sgpr(s64), [[UV15:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
- ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](s64), [[UV8]]
- ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](s64), [[UV9]]
+ ; GREEDY-NEXT: [[UV8:%[0-9]+]]:vgpr(i64), [[UV9:%[0-9]+]]:vgpr(i64), [[UV10:%[0-9]+]]:vgpr(i64), [[UV11:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x i32>)
+ ; GREEDY-NEXT: [[UV12:%[0-9]+]]:sgpr(i64), [[UV13:%[0-9]+]]:sgpr(i64), [[UV14:%[0-9]+]]:sgpr(i64), [[UV15:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<8 x i32>)
+ ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV12]](i64), [[UV8]]
+ ; GREEDY-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV13]](i64), [[UV9]]
; GREEDY-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](s64), [[UV10]]
+ ; GREEDY-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV14]](i64), [[UV10]]
; GREEDY-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](s64), [[UV11]]
+ ; GREEDY-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV15]](i64), [[UV11]]
; GREEDY-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[AND1]], [[ICMP3]]
; GREEDY-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32), [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT8:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV16]](i32)
@@ -499,14 +499,14 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsr
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT10:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV18]](i32)
; GREEDY-NEXT: [[INTRINSIC_CONVERGENT11:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV19]](i32)
; GREEDY-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT8]](i32), [[INTRINSIC_CONVERGENT9]](i32), [[INTRINSIC_CONVERGENT10]](i32), [[INTRINSIC_CONVERGENT11]](i32)
- ; GREEDY-NEXT: [[UV20:%[0-9]+]]:vgpr(s64), [[UV21:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GREEDY-NEXT: [[UV22:%[0-9]+]]:sgpr(s64), [[UV23:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<4 x i32>)
- ; GREEDY-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV22]](s64), [[UV20]]
+ ; GREEDY-NEXT: [[UV20:%[0-9]+]]:vgpr(i64), [[UV21:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GREEDY-NEXT: [[UV22:%[0-9]+]]:sgpr(i64), [[UV23:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<4 x i32>)
+ ; GREEDY-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV22]](i64), [[UV20]]
; GREEDY-NEXT: [[AND3:%[0-9]+]]:vcc(s1) = G_AND [[AND2]], [[ICMP4]]
- ; GREEDY-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV23]](s64), [[UV21]]
+ ; GREEDY-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV23]](i64), [[UV21]]
; GREEDY-NEXT: [[AND4:%[0-9]+]]:vcc(s1) = G_AND [[AND3]], [[ICMP5]]
- ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT12:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND4]](s1)
- ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT12]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GREEDY-NEXT: [[INTRINSIC_CONVERGENT12:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND4]](s1)
+ ; GREEDY-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GREEDY-NEXT: {{ $}}
; GREEDY-NEXT: bb.3:
; GREEDY-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
index e464fcf0e2428..ba5a021501756 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
@@ -13,8 +13,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), [[AMDGPU_COPY_VCC_SCC]](s1)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -49,8 +49,8 @@ legalized: true
body: |
bb.0:
; CHECK-LABEL: name: kill_constant_true
- ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](s32)
+ ; CHECK: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](i32)
; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), [[AMDGPU_COPY_VCC_SCC]](s1)
%0:_(s1) = G_CONSTANT i1 true
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), %0
@@ -63,8 +63,8 @@ legalized: true
body: |
bb.0:
; CHECK-LABEL: name: kill_constant_false
- ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](s32)
+ ; CHECK: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](i32)
; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), [[AMDGPU_COPY_VCC_SCC]](s1)
%0:_(s1) = G_CONSTANT i1 false
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
index 0673f063484d5..8c97df7d0e4fb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
@@ -75,13 +75,13 @@ define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -126,8 +126,8 @@ define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](i32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT]](i32), [[COPY5]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -176,16 +176,16 @@ define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](i32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY5]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll
index 18693a1185f30..cb20081e3e1ec 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.ptr.buffer.load.ll
@@ -74,13 +74,13 @@ define amdgpu_ps float @raw_ptr_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -125,8 +125,8 @@ define amdgpu_ps float @raw_ptr_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](i32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT]](i32), [[COPY5]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -175,16 +175,16 @@ define amdgpu_ps float @raw_ptr_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](i32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY5]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
index 581ed496b8229..0da15ed2ea424 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
@@ -354,9 +354,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY5]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -371,9 +371,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200_1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200_1250-NEXT: $vgpr0 = COPY [[COPY5]](f32)
; GFX1200_1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -392,9 +392,9 @@ define amdgpu_ps <2 x float> @s_buffer_load_v2f32_vgpr_offset(<4 x i32> inreg %r
; GFX7-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s64))
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x f32>) = COPY [[AMDGPU_BUFFER_LOAD]](<2 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[COPY5]](<2 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -411,9 +411,9 @@ define amdgpu_ps <2 x float> @s_buffer_load_v2f32_vgpr_offset(<4 x i32> inreg %r
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s64))
; GFX1200_1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x f32>) = COPY [[AMDGPU_BUFFER_LOAD]](<2 x f32>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[COPY5]](<2 x f32>)
; GFX1200_1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -434,9 +434,9 @@ define amdgpu_ps <3 x float> @s_buffer_load_v3f32_vgpr_offset(<4 x i32> inreg %r
; GFX7-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 16)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 16)
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x f32>) = COPY [[AMDGPU_BUFFER_LOAD]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[COPY5]](<4 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -454,9 +454,9 @@ define amdgpu_ps <3 x float> @s_buffer_load_v3f32_vgpr_offset(<4 x i32> inreg %r
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<3 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 16)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<3 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 16)
; GFX1200_1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(<3 x f32>) = COPY [[AMDGPU_BUFFER_LOAD]](<3 x f32>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[COPY5]](<3 x f32>)
; GFX1200_1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -478,9 +478,9 @@ define amdgpu_ps <4 x float> @s_buffer_load_v4f32_vgpr_offset(<4 x i32> inreg %r
; GFX7-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x f32>) = COPY [[AMDGPU_BUFFER_LOAD]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[COPY5]](<4 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -499,9 +499,9 @@ define amdgpu_ps <4 x float> @s_buffer_load_v4f32_vgpr_offset(<4 x i32> inreg %r
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128))
; GFX1200_1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x f32>) = COPY [[AMDGPU_BUFFER_LOAD]](<4 x f32>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[COPY5]](<4 x f32>)
; GFX1200_1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -524,10 +524,10 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset(<4 x i32> inreg %r
; GFX7-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -550,10 +550,10 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset(<4 x i32> inreg %r
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX1200_1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -580,12 +580,12 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset(<4 x i32> inreg
; GFX7-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -616,12 +616,12 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset(<4 x i32> inreg
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX1200_1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -657,9 +657,9 @@ define amdgpu_ps void @s_buffer_load_i96_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 8)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 8)
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(s128) = COPY [[AMDGPU_BUFFER_LOAD]](s128)
; GFX7-NEXT: [[TRUNC:%[0-9]+]]:vgpr(i96) = G_TRUNC [[COPY5]](s128)
; GFX7-NEXT: G_STORE [[TRUNC]](i96), [[DEF]](p1) :: (store (i96) into `ptr addrspace(1) poison`, align 8, addrspace 1)
@@ -676,9 +676,9 @@ define amdgpu_ps void @s_buffer_load_i96_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i96) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 8)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i96) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s96), align 8)
; GFX1200_1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i96) = COPY [[AMDGPU_BUFFER_LOAD]](i96)
; GFX1200_1250-NEXT: G_STORE [[COPY5]](i96), [[DEF]](p1) :: (store (i96) into `ptr addrspace(1) poison`, align 8, addrspace 1)
; GFX1200_1250-NEXT: S_ENDPGM 0
@@ -700,10 +700,10 @@ define amdgpu_ps void @s_buffer_load_i256_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
; GFX7-NEXT: [[MV:%[0-9]+]]:vgpr(i256) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i256)
; GFX7-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
@@ -723,10 +723,10 @@ define amdgpu_ps void @s_buffer_load_i256_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
; GFX1200_1250-NEXT: [[MV:%[0-9]+]]:vgpr(i256) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i256)
; GFX1200_1250-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
@@ -752,12 +752,12 @@ define amdgpu_ps void @s_buffer_load_i512_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 8)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48, align 8)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 8)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48, align 8)
; GFX7-NEXT: [[MV:%[0-9]+]]:vgpr(i512) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128), [[AMDGPU_BUFFER_LOAD2]](i128), [[AMDGPU_BUFFER_LOAD3]](i128)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128), [[UV2:%[0-9]+]]:vgpr(s128), [[UV3:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i512)
; GFX7-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
@@ -783,12 +783,12 @@ define amdgpu_ps void @s_buffer_load_i512_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 8)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48, align 8)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 8)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48, align 8)
; GFX1200_1250-NEXT: [[MV:%[0-9]+]]:vgpr(i512) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128), [[AMDGPU_BUFFER_LOAD2]](i128), [[AMDGPU_BUFFER_LOAD3]](i128)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128), [[UV2:%[0-9]+]]:vgpr(s128), [[UV3:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i512)
; GFX1200_1250-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
@@ -820,10 +820,10 @@ define amdgpu_ps void @s_buffer_load_v16i16_vgpr_offset(<4 x i32> inreg %rsrc, i
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x i16>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<8 x i16>), [[AMDGPU_BUFFER_LOAD1]](<8 x i16>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(<8 x i16>), [[UV1:%[0-9]+]]:vgpr(<8 x i16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x i16>)
; GFX7-NEXT: G_STORE [[UV]](<8 x i16>), [[DEF]](p1) :: (store (<8 x i16>) into `ptr addrspace(1) poison`, align 32, addrspace 1)
@@ -843,10 +843,10 @@ define amdgpu_ps void @s_buffer_load_v16i16_vgpr_offset(<4 x i32> inreg %rsrc, i
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x i16>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<8 x i16>), [[AMDGPU_BUFFER_LOAD1]](<8 x i16>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(<8 x i16>), [[UV1:%[0-9]+]]:vgpr(<8 x i16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x i16>)
; GFX1200_1250-NEXT: G_STORE [[UV]](<8 x i16>), [[DEF]](p1) :: (store (<8 x i16>) into `ptr addrspace(1) poison`, align 32, addrspace 1)
@@ -872,12 +872,12 @@ define amdgpu_ps void @s_buffer_load_v32i16_vgpr_offset(<4 x i32> inreg %rsrc, i
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<32 x i16>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<8 x i16>), [[AMDGPU_BUFFER_LOAD1]](<8 x i16>), [[AMDGPU_BUFFER_LOAD2]](<8 x i16>), [[AMDGPU_BUFFER_LOAD3]](<8 x i16>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(<8 x i16>), [[UV1:%[0-9]+]]:vgpr(<8 x i16>), [[UV2:%[0-9]+]]:vgpr(<8 x i16>), [[UV3:%[0-9]+]]:vgpr(<8 x i16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<32 x i16>)
; GFX7-NEXT: G_STORE [[UV]](<8 x i16>), [[DEF]](p1) :: (store (<8 x i16>) into `ptr addrspace(1) poison`, align 64, addrspace 1)
@@ -903,12 +903,12 @@ define amdgpu_ps void @s_buffer_load_v32i16_vgpr_offset(<4 x i32> inreg %rsrc, i
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<8 x i16>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<32 x i16>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<8 x i16>), [[AMDGPU_BUFFER_LOAD1]](<8 x i16>), [[AMDGPU_BUFFER_LOAD2]](<8 x i16>), [[AMDGPU_BUFFER_LOAD3]](<8 x i16>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(<8 x i16>), [[UV1:%[0-9]+]]:vgpr(<8 x i16>), [[UV2:%[0-9]+]]:vgpr(<8 x i16>), [[UV3:%[0-9]+]]:vgpr(<8 x i16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<32 x i16>)
; GFX1200_1250-NEXT: G_STORE [[UV]](<8 x i16>), [[DEF]](p1) :: (store (<8 x i16>) into `ptr addrspace(1) poison`, align 64, addrspace 1)
@@ -940,10 +940,10 @@ define amdgpu_ps void @s_buffer_load_v4i64_vgpr_offset(<4 x i32> inreg %rsrc, i3
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x i64>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x i64>), [[AMDGPU_BUFFER_LOAD1]](<2 x i64>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x i64>), [[UV1:%[0-9]+]]:vgpr(<2 x i64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x i64>)
; GFX7-NEXT: G_STORE [[UV]](<2 x i64>), [[DEF]](p1) :: (store (<2 x i64>) into `ptr addrspace(1) poison`, align 32, addrspace 1)
@@ -963,10 +963,10 @@ define amdgpu_ps void @s_buffer_load_v4i64_vgpr_offset(<4 x i32> inreg %rsrc, i3
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x i64>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x i64>), [[AMDGPU_BUFFER_LOAD1]](<2 x i64>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x i64>), [[UV1:%[0-9]+]]:vgpr(<2 x i64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x i64>)
; GFX1200_1250-NEXT: G_STORE [[UV]](<2 x i64>), [[DEF]](p1) :: (store (<2 x i64>) into `ptr addrspace(1) poison`, align 32, addrspace 1)
@@ -992,12 +992,12 @@ define amdgpu_ps void @s_buffer_load_v8i64_vgpr_offset(<4 x i32> inreg %rsrc, i3
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x i64>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x i64>), [[AMDGPU_BUFFER_LOAD1]](<2 x i64>), [[AMDGPU_BUFFER_LOAD2]](<2 x i64>), [[AMDGPU_BUFFER_LOAD3]](<2 x i64>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x i64>), [[UV1:%[0-9]+]]:vgpr(<2 x i64>), [[UV2:%[0-9]+]]:vgpr(<2 x i64>), [[UV3:%[0-9]+]]:vgpr(<2 x i64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x i64>)
; GFX7-NEXT: G_STORE [[UV]](<2 x i64>), [[DEF]](p1) :: (store (<2 x i64>) into `ptr addrspace(1) poison`, align 64, addrspace 1)
@@ -1023,12 +1023,12 @@ define amdgpu_ps void @s_buffer_load_v8i64_vgpr_offset(<4 x i32> inreg %rsrc, i3
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x i64>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x i64>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x i64>), [[AMDGPU_BUFFER_LOAD1]](<2 x i64>), [[AMDGPU_BUFFER_LOAD2]](<2 x i64>), [[AMDGPU_BUFFER_LOAD3]](<2 x i64>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x i64>), [[UV1:%[0-9]+]]:vgpr(<2 x i64>), [[UV2:%[0-9]+]]:vgpr(<2 x i64>), [[UV3:%[0-9]+]]:vgpr(<2 x i64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x i64>)
; GFX1200_1250-NEXT: G_STORE [[UV]](<2 x i64>), [[DEF]](p1) :: (store (<2 x i64>) into `ptr addrspace(1) poison`, align 64, addrspace 1)
@@ -1060,10 +1060,10 @@ define amdgpu_ps void @s_buffer_load_v4p1_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x p1>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x p1>), [[AMDGPU_BUFFER_LOAD1]](<2 x p1>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x p1>), [[UV1:%[0-9]+]]:vgpr(<2 x p1>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x p1>)
; GFX7-NEXT: G_STORE [[UV]](<2 x p1>), [[DEF]](p1) :: (store (<2 x p1>) into `ptr addrspace(1) poison`, align 32, addrspace 1)
@@ -1083,10 +1083,10 @@ define amdgpu_ps void @s_buffer_load_v4p1_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x p1>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x p1>), [[AMDGPU_BUFFER_LOAD1]](<2 x p1>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x p1>), [[UV1:%[0-9]+]]:vgpr(<2 x p1>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x p1>)
; GFX1200_1250-NEXT: G_STORE [[UV]](<2 x p1>), [[DEF]](p1) :: (store (<2 x p1>) into `ptr addrspace(1) poison`, align 32, addrspace 1)
@@ -1112,12 +1112,12 @@ define amdgpu_ps void @s_buffer_load_v8p1_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x p1>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x p1>), [[AMDGPU_BUFFER_LOAD1]](<2 x p1>), [[AMDGPU_BUFFER_LOAD2]](<2 x p1>), [[AMDGPU_BUFFER_LOAD3]](<2 x p1>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x p1>), [[UV1:%[0-9]+]]:vgpr(<2 x p1>), [[UV2:%[0-9]+]]:vgpr(<2 x p1>), [[UV3:%[0-9]+]]:vgpr(<2 x p1>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x p1>)
; GFX7-NEXT: G_STORE [[UV]](<2 x p1>), [[DEF]](p1) :: (store (<2 x p1>) into `ptr addrspace(1) poison`, align 64, addrspace 1)
@@ -1143,12 +1143,12 @@ define amdgpu_ps void @s_buffer_load_v8p1_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<2 x p1>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1200_1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x p1>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<2 x p1>), [[AMDGPU_BUFFER_LOAD1]](<2 x p1>), [[AMDGPU_BUFFER_LOAD2]](<2 x p1>), [[AMDGPU_BUFFER_LOAD3]](<2 x p1>)
; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x p1>), [[UV1:%[0-9]+]]:vgpr(<2 x p1>), [[UV2:%[0-9]+]]:vgpr(<2 x p1>), [[UV3:%[0-9]+]]:vgpr(<2 x p1>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x p1>)
; GFX1200_1250-NEXT: G_STORE [[UV]](<2 x p1>), [[DEF]](p1) :: (store (<2 x p1>) into `ptr addrspace(1) poison`, align 64, addrspace 1)
@@ -1181,9 +1181,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4092(<4 x i32> inreg %
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4092
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1201,9 +1201,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4092(<4 x i32> inreg %
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4092
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1221,9 +1221,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4092(<4 x i32> inreg %
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4092
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1246,8 +1246,8 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4095(<4 x i32> inreg %
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1265,9 +1265,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4095(<4 x i32> inreg %
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4095, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4095, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1285,9 +1285,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4095(<4 x i32> inreg %
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1310,8 +1310,8 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4096(<4 x i32> inreg %
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1329,9 +1329,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4096(<4 x i32> inreg %
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4096, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4096, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1349,9 +1349,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_offset_add_4096(<4 x i32> inreg %
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY6]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -1375,10 +1375,10 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_add_4064(<4 x i32>
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1404,10 +1404,10 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_add_4064(<4 x i32>
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX1200-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX1200-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1433,10 +1433,10 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_add_4064(<4 x i32>
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1468,9 +1468,9 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_add_4068(<4 x i32>
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4068
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1496,10 +1496,10 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_add_4068(<4 x i32>
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4068
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4068, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4084, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4068, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4084, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX1200-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX1200-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1525,10 +1525,10 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_add_4068(<4 x i32>
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4068
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>)
; GFX1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x f32>)
; GFX1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1559,12 +1559,12 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset_add_4032(<4 x i3
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4032
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4032, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4048, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4032, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4048, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1598,12 +1598,12 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset_add_4032(<4 x i3
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4032
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4032, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4048, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4032, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4048, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1200-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX1200-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX1200-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1637,12 +1637,12 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset_add_4032(<4 x i3
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4032
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1681,11 +1681,11 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset_add_4036(<4 x i3
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4036
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX7-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1719,12 +1719,12 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset_add_4036(<4 x i3
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4036
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4036, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4052, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4068, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4084, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4036, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4052, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4068, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4084, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1200-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX1200-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX1200-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1758,12 +1758,12 @@ define amdgpu_ps <16 x float> @s_buffer_load_v16f32_vgpr_offset_add_4036(<4 x i3
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4036
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 64)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48)
; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x f32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x f32>), [[AMDGPU_BUFFER_LOAD1]](<4 x f32>), [[AMDGPU_BUFFER_LOAD2]](<4 x f32>), [[AMDGPU_BUFFER_LOAD3]](<4 x f32>)
; GFX1250-NEXT: [[UV:%[0-9]+]]:vgpr(f32), [[UV1:%[0-9]+]]:vgpr(f32), [[UV2:%[0-9]+]]:vgpr(f32), [[UV3:%[0-9]+]]:vgpr(f32), [[UV4:%[0-9]+]]:vgpr(f32), [[UV5:%[0-9]+]]:vgpr(f32), [[UV6:%[0-9]+]]:vgpr(f32), [[UV7:%[0-9]+]]:vgpr(f32), [[UV8:%[0-9]+]]:vgpr(f32), [[UV9:%[0-9]+]]:vgpr(f32), [[UV10:%[0-9]+]]:vgpr(f32), [[UV11:%[0-9]+]]:vgpr(f32), [[UV12:%[0-9]+]]:vgpr(f32), [[UV13:%[0-9]+]]:vgpr(f32), [[UV14:%[0-9]+]]:vgpr(f32), [[UV15:%[0-9]+]]:vgpr(f32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x f32>)
; GFX1250-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1800,9 +1800,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc(<4 x i32> %rsrc, i32 inreg %
; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](i32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -1813,18 +1813,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc(<4 x i32> %rsrc, i32 inreg %
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY5]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY5]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -1846,9 +1846,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc(<4 x i32> %rsrc, i32 inreg %
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; GFX1200_1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](i32)
- ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
+ ; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200_1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200_1250-NEXT: {{ $}}
@@ -1859,18 +1859,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc(<4 x i32> %rsrc, i32 inreg %
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200_1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200_1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
; GFX1200_1250-NEXT: bb.3:
; GFX1200_1250-NEXT: successors: %bb.4, %bb.2
; GFX1200_1250-NEXT: {{ $}}
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY5]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY5]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200_1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200_1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
@@ -1899,7 +1899,7 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4092(<4 x i32> %
; GFX7-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4092
; GFX7-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -1910,18 +1910,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4092(<4 x i32> %
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -1945,7 +1945,7 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4092(<4 x i32> %
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4092
; GFX1200-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -1956,18 +1956,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4092(<4 x i32> %
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4092, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -1991,9 +1991,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4092(<4 x i32> %
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4092
; GFX1250-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -2004,18 +2004,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4092(<4 x i32> %
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -2045,9 +2045,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4096(<4 x i32> %
; GFX7-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX7-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -2058,18 +2058,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4096(<4 x i32> %
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -2093,7 +2093,7 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4096(<4 x i32> %
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX1200-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -2104,18 +2104,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4096(<4 x i32> %
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4096, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4096, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -2139,9 +2139,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4096(<4 x i32> %
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX1250-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -2152,18 +2152,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_soffset_add_4096(<4 x i32> %
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -2191,9 +2191,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4095(<4 x i32> %rsrc)
; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
- ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[C]](i32)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -2204,18 +2204,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4095(<4 x i32> %rsrc)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -2237,8 +2237,8 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4095(<4 x i32> %rsrc)
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200_1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200_1250-NEXT: {{ $}}
@@ -2249,18 +2249,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4095(<4 x i32> %rsrc)
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200_1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200_1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
; GFX1200_1250-NEXT: bb.3:
; GFX1200_1250-NEXT: successors: %bb.4, %bb.2
; GFX1200_1250-NEXT: {{ $}}
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[C2]], 4095, 0, 0 :: (dereferenceable invariant load (s32) from unknown-address + 4095, align 1)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[C2]], 4095, 0, 0 :: (dereferenceable invariant load (s32) from unknown-address + 4095, align 1)
; GFX1200_1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200_1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
@@ -2287,9 +2287,9 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4096(<4 x i32> %rsrc)
; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
- ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[C]](i32)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -2300,18 +2300,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4096(<4 x i32> %rsrc)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -2333,8 +2333,8 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4096(<4 x i32> %rsrc)
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200_1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200_1250-NEXT: {{ $}}
@@ -2345,18 +2345,18 @@ define amdgpu_ps float @s_buffer_load_f32_vgpr_rsrc_offset_4096(<4 x i32> %rsrc)
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200_1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200_1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
; GFX1200_1250-NEXT: bb.3:
; GFX1200_1250-NEXT: successors: %bb.4, %bb.2
; GFX1200_1250-NEXT: {{ $}}
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[C2]], 4096, 0, 0 :: (dereferenceable invariant load (s32) from unknown-address + 4096)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[C2]], 4096, 0, 0 :: (dereferenceable invariant load (s32) from unknown-address + 4096)
; GFX1200_1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200_1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
@@ -2386,7 +2386,7 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4064(<4 x i32> %
; GFX7-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
; GFX7-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -2397,19 +2397,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4064(<4 x i32> %
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -2441,7 +2441,7 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4064(<4 x i32> %
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
; GFX1200-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -2452,19 +2452,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4064(<4 x i32> %
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4064, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -2496,9 +2496,9 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4064(<4 x i32> %
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
; GFX1250-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -2509,19 +2509,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4064(<4 x i32> %
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -2560,9 +2560,9 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4068(<4 x i32> %
; GFX7-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4068
; GFX7-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -2573,19 +2573,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4068(<4 x i32> %
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -2617,7 +2617,7 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4068(<4 x i32> %
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4068
; GFX1200-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -2628,19 +2628,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4068(<4 x i32> %
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4068, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4084, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4068, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4084, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -2672,9 +2672,9 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4068(<4 x i32> %
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4068
; GFX1250-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -2685,19 +2685,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4068(<4 x i32> %
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -2734,9 +2734,9 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4096(<4 x i32> %
; GFX7-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX7-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -2747,19 +2747,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4096(<4 x i32> %
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -2791,7 +2791,7 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4096(<4 x i32> %
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX1200-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -2802,19 +2802,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4096(<4 x i32> %
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4096, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[COPY4]], 4112, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4096, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[COPY4]], 4112, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -2846,9 +2846,9 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4096(<4 x i32> %
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
; GFX1250-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[COPY4]], [[C]]
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](i32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[ADD]](i32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -2859,19 +2859,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_rsrc_add_4096(<4 x i32> %
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY5]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -2909,7 +2909,7 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_5000
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5000
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -2920,19 +2920,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_5000
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -2965,8 +2965,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_5000
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5000
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -2977,19 +2977,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_5000
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 5000, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 5016, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 5000, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 5016, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -3022,8 +3022,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_5000
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5000
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -3034,19 +3034,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_5000
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -3084,7 +3084,7 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4076
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4076
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -3095,19 +3095,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4076
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -3140,8 +3140,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4076
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4076
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -3152,19 +3152,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4076
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4076, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4092, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4076, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4092, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -3197,8 +3197,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4076
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4076
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -3209,19 +3209,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4076
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -3259,7 +3259,7 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4080
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4080
; GFX7-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -3270,19 +3270,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4080
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -3315,8 +3315,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4080
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4080
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200-NEXT: {{ $}}
@@ -3327,19 +3327,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4080
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.3:
; GFX1200-NEXT: successors: %bb.4, %bb.2
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[COPY4]], [[C1]], 4096, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4080, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[COPY4]], [[C1]], 4096, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1200-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200-NEXT: {{ $}}
@@ -3372,8 +3372,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4080
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4080
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY5]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
@@ -3384,19 +3384,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_add_4080
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: successors: %bb.4, %bb.2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 32)
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16)
; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1250-NEXT: {{ $}}
@@ -3431,8 +3431,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_offset_4
; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
- ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; GFX7-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX7-NEXT: {{ $}}
@@ -3443,19 +3443,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_offset_4
; GFX7-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX7-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX7-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX7-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX7-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX7-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: bb.3:
; GFX7-NEXT: successors: %bb.4, %bb.2
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[C2]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4064, align 32)
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[C2]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4080)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[C2]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4064, align 32)
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[C2]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4080)
; GFX7-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX7-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX7-NEXT: {{ $}}
@@ -3485,8 +3485,8 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_offset_4
; GFX1200_1250-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX1200_1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200_1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4064
- ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; GFX1200_1250-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GFX1200_1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1200_1250-NEXT: {{ $}}
@@ -3497,19 +3497,19 @@ define amdgpu_ps <8 x float> @s_buffer_load_v8f32_vgpr_offset_vgpr_rsrc_offset_4
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; GFX1200_1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; GFX1200_1250-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; GFX1200_1250-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; GFX1200_1250-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; GFX1200_1250-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](s32), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1200_1250-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_32_xm0_xexec(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; GFX1200_1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[INTRINSIC_CONVERGENT4]](i32), implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
; GFX1200_1250-NEXT: bb.3:
; GFX1200_1250-NEXT: successors: %bb.4, %bb.2
; GFX1200_1250-NEXT: {{ $}}
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[C2]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4064, align 32)
- ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](s32), [[C1]], [[C2]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4080)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[C2]], 4064, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4064, align 32)
+ ; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(<4 x f32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x i32>), [[C1]](i32), [[C1]], [[C2]], 4080, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 4080)
; GFX1200_1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX1200_1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX1200_1250-NEXT: {{ $}}
@@ -3546,8 +3546,8 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_vgpr_sgpr(<4 x i32> inreg %
; GFX7-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
; GFX7-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY6]]
- ; GFX7-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY7:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY7]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3565,8 +3565,8 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_vgpr_sgpr(<4 x i32> inreg %
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY6]]
- ; GFX1200-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY7]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3584,9 +3584,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_vgpr_sgpr(<4 x i32> inreg %
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY4]], [[COPY6]]
- ; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[ADD]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[ADD]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY7]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3609,8 +3609,8 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_sgpr_vgpr(<4 x i32> inreg %
; GFX7-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
; GFX7-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
; GFX7-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY6]], [[COPY4]]
- ; GFX7-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY7:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY7]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3628,8 +3628,8 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_sgpr_vgpr(<4 x i32> inreg %
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
; GFX1200-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY6]], [[COPY4]]
- ; GFX1200-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY7]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3647,9 +3647,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_sgpr_vgpr(<4 x i32> inreg %
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
; GFX1250-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[COPY6]], [[COPY4]]
- ; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](s32), [[ADD]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[ADD]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY7]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3675,9 +3675,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_vgpr_sgpr_imm(<4 x i32> inr
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX7-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3698,9 +3698,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_vgpr_sgpr_imm(<4 x i32> inr
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3721,9 +3721,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_vgpr_sgpr_imm(<4 x i32> inr
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3750,9 +3750,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_sgpr_vgpr_imm(<4 x i32> inr
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX7-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3773,9 +3773,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_sgpr_vgpr_imm(<4 x i32> inr
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3796,9 +3796,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_sgpr_vgpr_imm(<4 x i32> inr
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3826,9 +3826,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_imm_sgpr_vgpr(<4 x i32> inr
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX7-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3849,9 +3849,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_imm_sgpr_vgpr(<4 x i32> inr
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3872,9 +3872,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_imm_sgpr_vgpr(<4 x i32> inr
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3901,9 +3901,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_imm_vgpr_sgpr(<4 x i32> inr
; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX7-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX7-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX7-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX7-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3924,9 +3924,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_imm_vgpr_sgpr(<4 x i32> inr
; GFX1200-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1200-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1200-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD]], [[C1]], 1024, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1200-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -3947,9 +3947,9 @@ define amdgpu_ps float @s_buffer_load_f32_offset_add_imm_vgpr_sgpr(<4 x i32> inr
; GFX1250-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; GFX1250-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[COPY7]]
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](s32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(f32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C2]](i32), [[ADD1]], [[C1]], 0, 0, 0 :: (dereferenceable invariant load (s32))
; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr(f32) = COPY [[AMDGPU_BUFFER_LOAD]](f32)
; GFX1250-NEXT: $vgpr0 = COPY [[COPY8]](f32)
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
index 0e4da65e5975a..f620b91e623fd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
@@ -73,13 +73,13 @@ define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -123,8 +123,8 @@ define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex_vgp
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT]](i32), [[COPY6]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -172,16 +172,16 @@ define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
index 1f643c6b076ef..98872c5e1df52 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
@@ -74,13 +74,13 @@ define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -124,8 +124,8 @@ define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vg
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](i32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT]](i32), [[COPY7]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -173,16 +173,16 @@ define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](i32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](i32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](i32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY7]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll
index 6073ff1152ddd..615d7c15cbb8b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.load.ll
@@ -73,13 +73,13 @@ define amdgpu_ps float @struct_ptr_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -123,8 +123,8 @@ define amdgpu_ps float @struct_ptr_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT]](i32), [[COPY6]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -172,16 +172,16 @@ define amdgpu_ps float @struct_ptr_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll
index ae6ceda19502c..27143beed85ef 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.ptr.buffer.store.ll
@@ -74,13 +74,13 @@ define amdgpu_ps void @struct_ptr_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT4]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -124,8 +124,8 @@ define amdgpu_ps void @struct_ptr_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](i32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT]](i32), [[COPY7]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT1]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -173,16 +173,16 @@ define amdgpu_ps void @struct_ptr_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex
; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV2]](s32)
; CHECK-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV3]](s32)
; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32), [[INTRINSIC_CONVERGENT2]](s32), [[INTRINSIC_CONVERGENT3]](s32)
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
- ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(s64), [[UV7:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](s64), [[UV4]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](s64), [[UV5]]
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:vgpr(i64), [[UV5:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:sgpr(i64), [[UV7:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV6]](i64), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV7]](i64), [[UV5]]
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
; CHECK-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](i32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY7]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir
index 9c4d0d8c66139..edf302a3ce4f2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.wqm.demote.mir
@@ -13,8 +13,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.wqm.demote), [[AMDGPU_COPY_VCC_SCC]](s1)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -49,8 +49,8 @@ legalized: true
body: |
bb.0:
; CHECK-LABEL: name: wqm_demote_constant_true
- ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](s32)
+ ; CHECK: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](i32)
; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.wqm.demote), [[AMDGPU_COPY_VCC_SCC]](s1)
%0:_(s1) = G_CONSTANT i1 true
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.wqm.demote), %0
@@ -63,8 +63,8 @@ legalized: true
body: |
bb.0:
; CHECK-LABEL: name: wqm_demote_constant_false
- ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](s32)
+ ; CHECK: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C]](i32)
; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.wqm.demote), [[AMDGPU_COPY_VCC_SCC]](s1)
%0:_(s1) = G_CONSTANT i1 false
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.wqm.demote), %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir
index 632e19ac7b018..675d466969153 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbh-u32.mir
@@ -63,11 +63,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[AMDGPU_FFBH_U32_:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBH_U32 [[UV1]](s32)
- ; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBH_U32 [[UV]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 32
- ; CHECK-NEXT: [[UADDSAT:%[0-9]+]]:vgpr(s32) = G_UADDSAT [[AMDGPU_FFBH_U32_1]], [[C]]
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[AMDGPU_FFBH_U32_:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBH_U32 [[UV1]](i32)
+ ; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBH_U32 [[UV]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[UADDSAT:%[0-9]+]]:vgpr(i32) = G_UADDSAT [[AMDGPU_FFBH_U32_1]], [[C]]
; CHECK-NEXT: [[UMIN:%[0-9]+]]:vgpr(s32) = G_UMIN [[AMDGPU_FFBH_U32_]], [[UADDSAT]]
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_AMDGPU_FFBH_U32 %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir
index fe7210cf1654a..e5a52e350b30f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgpu-ffbl-b32.mir
@@ -63,11 +63,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[AMDGPU_FFBL_B32_:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBL_B32 [[UV]](s32)
- ; CHECK-NEXT: [[AMDGPU_FFBL_B32_1:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBL_B32 [[UV1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 32
- ; CHECK-NEXT: [[UADDSAT:%[0-9]+]]:vgpr(s32) = G_UADDSAT [[AMDGPU_FFBL_B32_1]], [[C]]
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[AMDGPU_FFBL_B32_:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBL_B32 [[UV]](i32)
+ ; CHECK-NEXT: [[AMDGPU_FFBL_B32_1:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBL_B32 [[UV1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[UADDSAT:%[0-9]+]]:vgpr(i32) = G_UADDSAT [[AMDGPU_FFBL_B32_1]], [[C]]
; CHECK-NEXT: [[UMIN:%[0-9]+]]:vgpr(s32) = G_UMIN [[AMDGPU_FFBL_B32_]], [[UADDSAT]]
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_AMDGPU_FFBL_B32 %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir
index 71adf63da145f..c76e49ad3324f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and-s1.mir
@@ -13,8 +13,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[COPY1]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[AND]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_TRUNC %0
@@ -37,10 +37,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[ICMP1]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[AND]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = G_CONSTANT i32 0
@@ -169,10 +169,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[COPY2]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[AMDGPU_COPY_VCC_SCC]], [[ICMP1]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $vgpr0
@@ -285,8 +285,8 @@ body: |
; CHECK-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY2]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[AND1]](s32)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[AND1]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[AMDGPU_COPY_VCC_SCC]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -311,9 +311,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[COPY1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[AND]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit [[AND1]](s32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[AND]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[AND1]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir
index d954ba050bd8f..379eecfb04191 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-anyext.mir
@@ -12,8 +12,8 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr(s32) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[DEF]](s32)
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[DEF]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s64) = G_ANYEXT %0
...
@@ -29,8 +29,8 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[DEF]](s32)
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[DEF]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s64) = G_ANYEXT %0
...
@@ -47,8 +47,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ICMP]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -67,8 +67,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[ICMP]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -88,9 +88,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr(s32) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[ICMP]](s32), [[DEF]](s32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[ICMP]](i32), [[DEF]](s32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -154,11 +154,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[DEF]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](i32), [[DEF]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -277,9 +277,11 @@ body: |
; CHECK-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[DEF]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](i32), [[DEF]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s1) = G_TRUNC %0
%2:_(s64) = G_ANYEXT %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
index a0cb85f710443..131e38ba5e906 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
@@ -97,10 +97,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ASHR]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(i32) = G_ASHR [[SEXT]], [[ZEXT]](i32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ASHR]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC2]](s16)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -199,16 +199,16 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[BITCAST]], 16
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[BITCAST1]], 16
- ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:sgpr(s32) = G_ASHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:sgpr(s32) = G_ASHR [[SEXT_INREG]], [[SEXT_INREG1]](s32)
- ; CHECK-NEXT: [[ASHR3:%[0-9]+]]:sgpr(s32) = G_ASHR [[ASHR]], [[ASHR1]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ASHR2]](s32), [[ASHR3]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(i32) = G_SEXT_INREG [[BITCAST]], 16
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(i32) = G_ASHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:sgpr(i32) = G_SEXT_INREG [[BITCAST1]], 16
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:sgpr(i32) = G_ASHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:sgpr(i32) = G_ASHR [[SEXT_INREG]], [[SEXT_INREG1]](i32)
+ ; CHECK-NEXT: [[ASHR3:%[0-9]+]]:sgpr(i32) = G_ASHR [[ASHR]], [[ASHR1]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ASHR2]](i32), [[ASHR3]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR_TRUNC]](<2 x s16>)
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
index cd957c8b9c464..0923c795ba18d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
@@ -38,10 +38,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: G_BRCOND [[AND]](s32), %bb.1
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: G_BRCOND [[AND]](i32), %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
bb.0.entry:
@@ -65,9 +65,9 @@ body: |
; CHECK-NEXT: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: G_BRCOND [[AND]](s32), %bb.1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: G_BRCOND [[AND]](i32), %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
bb.0.entry:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-call.ll
index 919811e3e83ba..4b10c4b6015a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-call.ll
@@ -40,8 +40,8 @@ define amdgpu_ps void @test_divergent_indirect_call_p0(ptr %fptr) {
; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
; CHECK-NEXT: [[MV1:%[0-9]+]]:sgpr(p0) = G_MERGE_VALUES [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[MV1]](p0), [[MV]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
@@ -86,8 +86,8 @@ define amdgpu_ps i32 @test_divergent_indirect_call_p0_with_args(ptr %fptr, i32 %
; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
; CHECK-NEXT: [[MV1:%[0-9]+]]:sgpr(p0) = G_MERGE_VALUES [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[MV1]](p0), [[MV]]
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-poison.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-poison.mir
index 678de17f777d6..146db88155b38 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-poison.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctlz-zero-poison.mir
@@ -66,11 +66,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[AMDGPU_FFBH_U32_:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBH_U32 [[UV1]](s32)
- ; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBH_U32 [[UV]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 32
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[AMDGPU_FFBH_U32_1]], [[C]]
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[AMDGPU_FFBH_U32_:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBH_U32 [[UV1]](i32)
+ ; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBH_U32 [[UV]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[AMDGPU_FFBH_U32_1]], [[C]]
; CHECK-NEXT: [[UMIN:%[0-9]+]]:vgpr(s32) = G_UMIN [[AMDGPU_FFBH_U32_]], [[ADD]]
; CHECK-NEXT: S_ENDPGM 0, implicit [[UMIN]](s32)
%0:_(s64) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir
index 4acc4d794d9f7..6a0037f8c8dde 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ctpop.mir
@@ -68,9 +68,9 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[CTPOP:%[0-9]+]]:vgpr(s32) = G_CTPOP [[UV]](s32)
- ; CHECK-NEXT: [[CTPOP1:%[0-9]+]]:vgpr(s32) = G_CTPOP [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[CTPOP:%[0-9]+]]:vgpr(i32) = G_CTPOP [[UV]](i32)
+ ; CHECK-NEXT: [[CTPOP1:%[0-9]+]]:vgpr(i32) = G_CTPOP [[UV1]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = nuw nsw G_ADD [[CTPOP]], [[CTPOP1]]
; CHECK-NEXT: S_ENDPGM 0, implicit [[ADD]](s32)
%0:_(s64) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-poison.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-poison.mir
index 531b6138bb685..de6e9b7327c6d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-poison.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-cttz-zero-poison.mir
@@ -66,11 +66,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[AMDGPU_FFBL_B32_:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBL_B32 [[UV]](s32)
- ; CHECK-NEXT: [[AMDGPU_FFBL_B32_1:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FFBL_B32 [[UV1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 32
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[AMDGPU_FFBL_B32_1]], [[C]]
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[AMDGPU_FFBL_B32_:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBL_B32 [[UV]](i32)
+ ; CHECK-NEXT: [[AMDGPU_FFBL_B32_1:%[0-9]+]]:vgpr(i32) = G_AMDGPU_FFBL_B32 [[UV1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[AMDGPU_FFBL_B32_1]], [[C]]
; CHECK-NEXT: [[UMIN:%[0-9]+]]:vgpr(s32) = G_UMIN [[AMDGPU_FFBL_B32_]], [[ADD]]
; CHECK-NEXT: S_ENDPGM 0, implicit [[UMIN]](s32)
%0:_(s64) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir
index 1416b0bc5d4ba..c58993f35f64d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-dyn-stackalloc.mir
@@ -18,10 +18,10 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
;
@@ -29,10 +29,10 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
%0:_(s32) = COPY $sgpr0
@@ -55,10 +55,10 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
;
@@ -66,10 +66,10 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
%0:_(s32) = COPY $sgpr0
@@ -92,10 +92,10 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
;
@@ -103,10 +103,10 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
%0:_(s32) = COPY $sgpr0
@@ -129,10 +129,10 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
;
@@ -140,10 +140,10 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
%0:_(s32) = COPY $sgpr0
@@ -166,10 +166,10 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
;
@@ -177,10 +177,10 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
%0:_(s32) = COPY $sgpr0
@@ -203,14 +203,14 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2047
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -2048
- ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2047
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -2048
+ ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
;
@@ -218,14 +218,14 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1023
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1024
- ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1023
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -1024
+ ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
%0:_(s32) = COPY $sgpr0
@@ -248,14 +248,14 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4095
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -4096
- ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -4096
+ ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
;
@@ -263,14 +263,14 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2047
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -2048
- ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2047
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -2048
+ ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
%0:_(s32) = COPY $sgpr0
@@ -293,14 +293,14 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 8191
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -8192
- ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 8191
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -8192
+ ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
;
@@ -308,14 +308,14 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY]], [[C]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4095
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -4096
- ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -4096
+ ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
%0:_(s32) = COPY $sgpr0
@@ -335,19 +335,19 @@ body: |
; WAVE64-LABEL: name: test_dyn_stackalloc_sgpr_constant_align4
; WAVE64: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2048
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2048
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY]](p5)
;
; WAVE32-LABEL: name: test_dyn_stackalloc_sgpr_constant_align4
; WAVE32: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1024
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY]](p5)
%0:_(s32) = G_CONSTANT i32 32
@@ -370,10 +370,10 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2048
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2048
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY]](p5)
;
@@ -381,10 +381,10 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1024
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY]](p5)
%0:_(s32) = G_CONSTANT i32 32
@@ -407,10 +407,10 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2048
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2048
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY]](p5)
;
@@ -418,10 +418,10 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1024
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C2]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY]](p5)
%0:_(s32) = G_CONSTANT i32 32
@@ -444,14 +444,14 @@ body: |
; WAVE64: liveins: $sgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2048
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2048
; WAVE64-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2047
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
- ; WAVE64-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -2048
- ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C4]](s32)
- ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[C2]](s32)
+ ; WAVE64-NEXT: [[C3:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2047
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C3]](i32)
+ ; WAVE64-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -2048
+ ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C4]](i32)
+ ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[C2]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
;
@@ -459,14 +459,14 @@ body: |
; WAVE32: liveins: $sgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 32
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1024
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1024
; WAVE32-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1023
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
- ; WAVE32-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1024
- ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C4]](s32)
- ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[C2]](s32)
+ ; WAVE32-NEXT: [[C3:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1023
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C3]](i32)
+ ; WAVE32-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -1024
+ ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C4]](i32)
+ ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[C2]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
%0:_(s32) = G_CONSTANT i32 32
@@ -489,11 +489,11 @@ body: |
; WAVE64: liveins: $vgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; WAVE64-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](s32)
+ ; WAVE64-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
;
@@ -501,11 +501,11 @@ body: |
; WAVE32: liveins: $vgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; WAVE32-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](s32)
+ ; WAVE32-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
%0:_(s32) = COPY $vgpr0
@@ -528,11 +528,11 @@ body: |
; WAVE64: liveins: $vgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; WAVE64-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](s32)
+ ; WAVE64-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
;
@@ -540,11 +540,11 @@ body: |
; WAVE32: liveins: $vgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; WAVE32-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](s32)
+ ; WAVE32-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[COPY1]](p5)
%0:_(s32) = COPY $vgpr0
@@ -567,15 +567,15 @@ body: |
; WAVE64: liveins: $vgpr0
; WAVE64-NEXT: {{ $}}
; WAVE64-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; WAVE64-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](s32)
+ ; WAVE64-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 6
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](i32)
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4095
- ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -4096
- ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4095
+ ; WAVE64-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -4096
+ ; WAVE64-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE64-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE64-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE64-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
;
@@ -583,15 +583,15 @@ body: |
; WAVE32: liveins: $vgpr0
; WAVE32-NEXT: {{ $}}
; WAVE32-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; WAVE32-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](s32)
+ ; WAVE32-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.wave.reduce.umax), [[COPY]](s32), 0
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 5
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[INTRINSIC_CONVERGENT]], [[C]](i32)
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(p5) = COPY $sp_reg
- ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2047
- ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](s32)
- ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -2048
- ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](s32)
- ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](s32)
+ ; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 2047
+ ; WAVE32-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY1]], [[C1]](i32)
+ ; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -2048
+ ; WAVE32-NEXT: [[PTRMASK:%[0-9]+]]:sgpr(p5) = G_PTRMASK [[PTR_ADD]], [[C2]](i32)
+ ; WAVE32-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[PTRMASK]], [[SHL]](i32)
; WAVE32-NEXT: $sp_reg = COPY [[PTR_ADD1]](p5)
; WAVE32-NEXT: S_ENDPGM 0, implicit [[PTRMASK]](p5)
%0:_(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir
index 33c545c634772..1d50874e93bf6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract-vector-elt.mir
@@ -354,12 +354,12 @@ body: |
; WAVE64-NEXT: [[COPY:%[0-9]+]]:vgpr(<8 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; WAVE64-NEXT: [[BITCAST:%[0-9]+]]:vgpr(<16 x s32>) = G_BITCAST [[COPY]](<8 x s64>)
- ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY1]], [[C]](s32)
- ; WAVE64-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[SHL]], [[C]]
- ; WAVE64-NEXT: [[EVEC:%[0-9]+]]:vgpr(s32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[SHL]](s32)
- ; WAVE64-NEXT: [[EVEC1:%[0-9]+]]:vgpr(s32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[ADD]](s32)
- ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[EVEC]](s32), [[EVEC1]](s32)
+ ; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; WAVE64-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY1]], [[C]](i32)
+ ; WAVE64-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[SHL]], [[C]]
+ ; WAVE64-NEXT: [[EVEC:%[0-9]+]]:vgpr(i32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[SHL]](i32)
+ ; WAVE64-NEXT: [[EVEC1:%[0-9]+]]:vgpr(i32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[ADD]](i32)
+ ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[EVEC]](i32), [[EVEC1]](i32)
; WAVE64-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
;
; WAVE32-LABEL: name: extract_vector_elt_v8s64_vs
@@ -368,12 +368,12 @@ body: |
; WAVE32-NEXT: [[COPY:%[0-9]+]]:vgpr(<8 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; WAVE32-NEXT: [[BITCAST:%[0-9]+]]:vgpr(<16 x s32>) = G_BITCAST [[COPY]](<8 x s64>)
- ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY1]], [[C]](s32)
- ; WAVE32-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[SHL]], [[C]]
- ; WAVE32-NEXT: [[EVEC:%[0-9]+]]:vgpr(s32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[SHL]](s32)
- ; WAVE32-NEXT: [[EVEC1:%[0-9]+]]:vgpr(s32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[ADD]](s32)
- ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[EVEC]](s32), [[EVEC1]](s32)
+ ; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; WAVE32-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY1]], [[C]](i32)
+ ; WAVE32-NEXT: [[ADD:%[0-9]+]]:sgpr(i32) = G_ADD [[SHL]], [[C]]
+ ; WAVE32-NEXT: [[EVEC:%[0-9]+]]:vgpr(i32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[SHL]](i32)
+ ; WAVE32-NEXT: [[EVEC1:%[0-9]+]]:vgpr(i32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<16 x s32>), [[ADD]](i32)
+ ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[EVEC]](i32), [[EVEC1]](i32)
; WAVE32-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(<8 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
%1:_(s32) = COPY $sgpr0
@@ -397,43 +397,43 @@ body: |
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; WAVE64-NEXT: [[COPY2:%[0-9]+]]:vgpr(<8 x s64>) = COPY [[COPY]](<8 x s64>)
; WAVE64-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY2]](<8 x s64>)
- ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; WAVE64-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
- ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE64-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
- ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE64-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C2]]
- ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE64-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE64-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C3]]
- ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE64-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE64-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C4]]
- ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE64-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE64-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C5]]
- ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE64-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE64-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C6]]
- ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE64-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
;
; WAVE32-LABEL: name: extract_vector_elt_v8s64_sv
@@ -443,43 +443,43 @@ body: |
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; WAVE32-NEXT: [[COPY2:%[0-9]+]]:vgpr(<8 x s64>) = COPY [[COPY]](<8 x s64>)
; WAVE32-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY2]](<8 x s64>)
- ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; WAVE32-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
- ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE32-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
- ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE32-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C2]]
- ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE32-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE32-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C3]]
- ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE32-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE32-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C4]]
- ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE32-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE32-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C5]]
- ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE32-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE32-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C6]]
- ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE32-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(<8 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
%1:_(s32) = COPY $vgpr0
@@ -502,43 +502,43 @@ body: |
; WAVE64-NEXT: [[COPY:%[0-9]+]]:vgpr(<8 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; WAVE64-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr16
; WAVE64-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY]](<8 x s64>)
- ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE64-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; WAVE64-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
- ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE64-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
- ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE64-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C2]]
- ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE64-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE64-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C3]]
- ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE64-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE64-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C4]]
- ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE64-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE64-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C5]]
- ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE64-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE64-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C6]]
- ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE64-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
;
; WAVE32-LABEL: name: extract_vector_elt_v8s64_vv
@@ -547,43 +547,43 @@ body: |
; WAVE32-NEXT: [[COPY:%[0-9]+]]:vgpr(<8 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; WAVE32-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr16
; WAVE32-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY]](<8 x s64>)
- ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE32-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; WAVE32-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
- ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE32-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
- ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE32-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C2]]
- ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE32-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE32-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C3]]
- ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE32-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE32-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C4]]
- ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE32-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE32-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C5]]
- ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE32-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE32-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C6]]
- ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE32-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(<8 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
%1:_(s32) = COPY $vgpr16
@@ -997,42 +997,42 @@ body: |
; WAVE64-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; WAVE64-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[COPY1]], [[COPY2]]
; WAVE64-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY]](<8 x s64>)
- ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE64-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C]]
- ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE64-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C1]]
- ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE64-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C2]]
- ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE64-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE64-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C3]]
- ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE64-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE64-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C4]]
- ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE64-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE64-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C5]]
- ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE64-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE64-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C6]]
- ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE64-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
;
; WAVE32-LABEL: name: extract_vector_elt_v8s64_vv_idx_add1
@@ -1044,42 +1044,42 @@ body: |
; WAVE32-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; WAVE32-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[COPY1]], [[COPY2]]
; WAVE32-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY]](<8 x s64>)
- ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE32-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C]]
- ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE32-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C1]]
- ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE32-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C2]]
- ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE32-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE32-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C3]]
- ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE32-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE32-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C4]]
- ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE32-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE32-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C5]]
- ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE32-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE32-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C6]]
- ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE32-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(<8 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
%1:_(s32) = COPY $vgpr16
@@ -1238,42 +1238,42 @@ body: |
; WAVE64-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[COPY1]], [[COPY2]]
; WAVE64-NEXT: [[COPY3:%[0-9]+]]:vgpr(<8 x s64>) = COPY [[COPY]](<8 x s64>)
; WAVE64-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY3]](<8 x s64>)
- ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE64-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE64-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C]]
- ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE64-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE64-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE64-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE64-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE64-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C1]]
- ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE64-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE64-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE64-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE64-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE64-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C2]]
- ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE64-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE64-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE64-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE64-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE64-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C3]]
- ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE64-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE64-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE64-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE64-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE64-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C4]]
- ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE64-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE64-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE64-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE64-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE64-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C5]]
- ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE64-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE64-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE64-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE64-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE64-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C6]]
- ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE64-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE64-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE64-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE64-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE64-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
;
; WAVE32-LABEL: name: extract_vector_elt_v8s64_sv_add1
@@ -1286,42 +1286,42 @@ body: |
; WAVE32-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[COPY1]], [[COPY2]]
; WAVE32-NEXT: [[COPY3:%[0-9]+]]:vgpr(<8 x s64>) = COPY [[COPY]](<8 x s64>)
; WAVE32-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[COPY3]](<8 x s64>)
- ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; WAVE32-NEXT: [[UV8:%[0-9]+]]:vgpr(i32), [[UV9:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV]](s64)
; WAVE32-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C]]
- ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV1]](s64)
- ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
- ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
+ ; WAVE32-NEXT: [[UV10:%[0-9]+]]:vgpr(i32), [[UV11:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; WAVE32-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV10]], [[UV8]]
+ ; WAVE32-NEXT: [[SELECT1:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[UV11]], [[UV9]]
; WAVE32-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; WAVE32-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C1]]
- ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV2]](s64)
- ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
- ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
+ ; WAVE32-NEXT: [[UV12:%[0-9]+]]:vgpr(i32), [[UV13:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV2]](s64)
+ ; WAVE32-NEXT: [[SELECT2:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT]]
+ ; WAVE32-NEXT: [[SELECT3:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT1]]
; WAVE32-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 3
; WAVE32-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C2]]
- ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV3]](s64)
- ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
- ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
+ ; WAVE32-NEXT: [[UV14:%[0-9]+]]:vgpr(i32), [[UV15:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV3]](s64)
+ ; WAVE32-NEXT: [[SELECT4:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV14]], [[SELECT2]]
+ ; WAVE32-NEXT: [[SELECT5:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP2]](s1), [[UV15]], [[SELECT3]]
; WAVE32-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; WAVE32-NEXT: [[ICMP3:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C3]]
- ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(s32), [[UV17:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV4]](s64)
- ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
- ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
+ ; WAVE32-NEXT: [[UV16:%[0-9]+]]:vgpr(i32), [[UV17:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV4]](s64)
+ ; WAVE32-NEXT: [[SELECT6:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV16]], [[SELECT4]]
+ ; WAVE32-NEXT: [[SELECT7:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP3]](s1), [[UV17]], [[SELECT5]]
; WAVE32-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 5
; WAVE32-NEXT: [[ICMP4:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C4]]
- ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(s32), [[UV19:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV5]](s64)
- ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
- ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
+ ; WAVE32-NEXT: [[UV18:%[0-9]+]]:vgpr(i32), [[UV19:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV5]](s64)
+ ; WAVE32-NEXT: [[SELECT8:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV18]], [[SELECT6]]
+ ; WAVE32-NEXT: [[SELECT9:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP4]](s1), [[UV19]], [[SELECT7]]
; WAVE32-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
; WAVE32-NEXT: [[ICMP5:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C5]]
- ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(s32), [[UV21:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV6]](s64)
- ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
- ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
+ ; WAVE32-NEXT: [[UV20:%[0-9]+]]:vgpr(i32), [[UV21:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV6]](s64)
+ ; WAVE32-NEXT: [[SELECT10:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV20]], [[SELECT8]]
+ ; WAVE32-NEXT: [[SELECT11:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP5]](s1), [[UV21]], [[SELECT9]]
; WAVE32-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 7
; WAVE32-NEXT: [[ICMP6:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[ADD]](s32), [[C6]]
- ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(s32), [[UV23:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV7]](s64)
- ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
- ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
- ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](s32), [[SELECT13]](s32)
+ ; WAVE32-NEXT: [[UV22:%[0-9]+]]:vgpr(i32), [[UV23:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[UV7]](s64)
+ ; WAVE32-NEXT: [[SELECT12:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV22]], [[SELECT10]]
+ ; WAVE32-NEXT: [[SELECT13:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP6]](s1), [[UV23]], [[SELECT11]]
+ ; WAVE32-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT12]](i32), [[SELECT13]](i32)
; WAVE32-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(<8 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
%1:_(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir
index da8fdd23d38fd..2c066ceb7f5ce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcanonicalize.mir
@@ -15,8 +15,8 @@ body: |
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[TRUNC]](s16)
; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:vgpr(s16) = G_FCANONICALIZE [[COPY1]]
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[FCANONICALIZE]](s16)
- ; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[FCANONICALIZE]](s16)
+ ; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s16) = G_TRUNC %0
%2:_(s16) = G_FCANONICALIZE %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
index 956f3342f56af..335e077fb9abe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
@@ -23,7 +23,7 @@ body: |
; GFX1150-NEXT: {{ $}}
; GFX1150-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; GFX1150-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; GFX1150-NEXT: [[FCMP:%[0-9]+]]:sgpr(s32) = G_FCMP floatpred(uge), [[COPY]](s32), [[COPY1]]
+ ; GFX1150-NEXT: [[FCMP:%[0-9]+]]:sgpr(i32) = G_FCMP floatpred(uge), [[COPY]](s32), [[COPY1]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_FCMP floatpred(uge), %0(s32), %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
index 61e46068aa9b3..2055766402fd0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
@@ -51,8 +51,8 @@ body: |
; GFX8-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[TRUNC]](s16)
; GFX8-NEXT: [[INT:%[0-9]+]]:vgpr(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.exp2), [[COPY1]](s16)
- ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[INT]](s16)
- ; GFX8-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[INT]](s16)
+ ; GFX8-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
;
; GFX11-LABEL: name: amdgcn_exp2_s16_s
; GFX11: liveins: $sgpr0
@@ -61,8 +61,8 @@ body: |
; GFX11-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[TRUNC]](s16)
; GFX11-NEXT: [[INT:%[0-9]+]]:vgpr(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.exp2), [[COPY1]](s16)
- ; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[INT]](s16)
- ; GFX11-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[INT]](s16)
+ ; GFX11-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
;
; GFX12-LABEL: name: amdgcn_exp2_s16_s
; GFX12: liveins: $sgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir
index d511b4a0c90e3..09a108fcd909b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-freeze.mir
@@ -66,8 +66,8 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:sgpr(s32) = G_FREEZE [[COPY]]
- ; CHECK-NEXT: $sgpr0 = COPY [[FREEZE]](s32)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:sgpr(i32) = G_FREEZE [[COPY]]
+ ; CHECK-NEXT: $sgpr0 = COPY [[FREEZE]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s1) = G_TRUNC %0(s32)
%2:_(s1) = G_FREEZE %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
index 5b4275bbd250d..30d47f6b748da 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
@@ -16,8 +16,8 @@ body: |
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[TRUNC]](s16)
; GFX9-NEXT: [[FSQRT:%[0-9]+]]:vgpr(s16) = G_FSQRT [[COPY1]]
- ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[FSQRT]](s16)
- ; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[FSQRT]](s16)
+ ; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
;
; GFX12-LABEL: name: fsqrt_s16_s
; GFX12: liveins: $sgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir
index 02d7072de00b2..23871aee2e993 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.s16.mir
@@ -15,9 +15,9 @@ body: |
; GFX8-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; GFX8-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; GFX8-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[ZEXT]](s32), [[ZEXT1]]
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; GFX8-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[ZEXT]](i32), [[ZEXT1]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s16) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
index 9034a94535005..1412e314f3a05 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
@@ -120,8 +120,8 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[COPY]](p1) :: (load (<4 x s32>) from %ir.global.not.uniform.v8i32, align 32, addrspace 1)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p1) :: (load (<4 x s32>) from %ir.global.not.uniform.v8i32 + 16, basealign 32, addrspace 1)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x s32>)
@@ -152,8 +152,8 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[COPY]](p1) :: (load (<2 x s64>) from %ir.global.not.uniform.v4i64, align 32, addrspace 1)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD]](p1) :: (load (<2 x s64>) from %ir.global.not.uniform.v4i64 + 16, basealign 32, addrspace 1)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x s64>)
@@ -191,14 +191,14 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[COPY]](p1) :: (load (<4 x s32>) from %ir.global.not.uniform.v16i32, align 64, addrspace 1)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p1) :: (load (<4 x s32>) from %ir.global.not.uniform.v16i32 + 16, basealign 64, addrspace 1)
- ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 32
- ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+ ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
+ ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; GCN-NEXT: [[LOAD2:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD1]](p1) :: (load (<4 x s32>) from %ir.global.not.uniform.v16i32 + 32, align 32, basealign 64, addrspace 1)
- ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 48
- ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](s64)
+ ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
+ ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; GCN-NEXT: [[LOAD3:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD2]](p1) :: (load (<4 x s32>) from %ir.global.not.uniform.v16i32 + 48, basealign 64, addrspace 1)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32), [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32), [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32), [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32), [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x s32>)
@@ -236,14 +236,14 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[COPY]](p1) :: (load (<2 x s64>) from %ir.global.not.uniform.v8i64, align 64, addrspace 1)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD]](p1) :: (load (<2 x s64>) from %ir.global.not.uniform.v8i64 + 16, basealign 64, addrspace 1)
- ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 32
- ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+ ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
+ ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; GCN-NEXT: [[LOAD2:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD1]](p1) :: (load (<2 x s64>) from %ir.global.not.uniform.v8i64 + 32, align 32, basealign 64, addrspace 1)
- ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 48
- ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](s64)
+ ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
+ ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; GCN-NEXT: [[LOAD3:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD2]](p1) :: (load (<2 x s64>) from %ir.global.not.uniform.v8i64 + 48, basealign 64, addrspace 1)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>), [[LOAD2]](<2 x s64>), [[LOAD3]](<2 x s64>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x s64>)
@@ -365,8 +365,8 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[COPY]](p4) :: (load (<4 x s32>) from %ir.constant.not.uniform.v8i32, align 32, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p4) :: (load (<4 x s32>) from %ir.constant.not.uniform.v8i32 + 16, basealign 32, addrspace 4)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x s32>)
@@ -395,11 +395,11 @@ body: |
; GCN: liveins: $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
- ; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p4) :: (load (s128) from %ir.constant.not.uniform, align 32, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
- ; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(s128) = G_LOAD [[PTR_ADD]](p4) :: (load (s128) from %ir.constant.not.uniform + 16, basealign 32, addrspace 4)
- ; GCN-NEXT: [[MV:%[0-9]+]]:vgpr(s256) = G_MERGE_VALUES [[LOAD]](s128), [[LOAD1]](s128)
+ ; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(i128) = G_LOAD [[COPY]](p4) :: (load (i128) from %ir.constant.not.uniform, align 32, addrspace 4)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
+ ; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(i128) = G_LOAD [[PTR_ADD]](p4) :: (load (i128) from %ir.constant.not.uniform + 16, basealign 32, addrspace 4)
+ ; GCN-NEXT: [[MV:%[0-9]+]]:vgpr(s256) = G_MERGE_VALUES [[LOAD]](i128), [[LOAD1]](i128)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s256)
; GCN-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[UV]]
; GCN-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[UV1]]
@@ -428,8 +428,8 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p4) :: (load (<8 x s16>) from %ir.constant.not.uniform, align 32, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[PTR_ADD]](p4) :: (load (<8 x s16>) from %ir.constant.not.uniform + 16, basealign 32, addrspace 4)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x s16>) = G_CONCAT_VECTORS [[LOAD]](<8 x s16>), [[LOAD1]](<8 x s16>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x s16>), [[UV1:%[0-9]+]]:vgpr(<2 x s16>), [[UV2:%[0-9]+]]:vgpr(<2 x s16>), [[UV3:%[0-9]+]]:vgpr(<2 x s16>), [[UV4:%[0-9]+]]:vgpr(<2 x s16>), [[UV5:%[0-9]+]]:vgpr(<2 x s16>), [[UV6:%[0-9]+]]:vgpr(<2 x s16>), [[UV7:%[0-9]+]]:vgpr(<2 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x s16>)
@@ -459,8 +459,8 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[COPY]](p4) :: (load (<2 x s64>) from %ir.constant.not.uniform.v4i64, align 32, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD]](p4) :: (load (<2 x s64>) from %ir.constant.not.uniform.v4i64 + 16, basealign 32, addrspace 4)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x s64>)
@@ -498,14 +498,14 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[COPY]](p4) :: (load (<4 x s32>) from %ir.constant.not.uniform.v16i32, align 64, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p4) :: (load (<4 x s32>) from %ir.constant.not.uniform.v16i32 + 16, basealign 64, addrspace 4)
- ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 32
- ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+ ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
+ ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; GCN-NEXT: [[LOAD2:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (load (<4 x s32>) from %ir.constant.not.uniform.v16i32 + 32, align 32, basealign 64, addrspace 4)
- ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 48
- ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](s64)
+ ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
+ ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; GCN-NEXT: [[LOAD3:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD2]](p4) :: (load (<4 x s32>) from %ir.constant.not.uniform.v16i32 + 48, basealign 64, addrspace 4)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32), [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32), [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32), [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32), [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x s32>)
@@ -543,14 +543,14 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[COPY]](p4) :: (load (<2 x s64>) from %ir.constant.not.uniform.v8i64, align 64, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD]](p4) :: (load (<2 x s64>) from %ir.constant.not.uniform.v8i64 + 16, basealign 64, addrspace 4)
- ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 32
- ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+ ; GCN-NEXT: [[C1:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
+ ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; GCN-NEXT: [[LOAD2:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD1]](p4) :: (load (<2 x s64>) from %ir.constant.not.uniform.v8i64 + 32, align 32, basealign 64, addrspace 4)
- ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 48
- ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](s64)
+ ; GCN-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
+ ; GCN-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; GCN-NEXT: [[LOAD3:%[0-9]+]]:vgpr(<2 x s64>) = G_LOAD [[PTR_ADD2]](p4) :: (load (<2 x s64>) from %ir.constant.not.uniform.v8i64 + 48, basealign 64, addrspace 4)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>), [[LOAD2]](<2 x s64>), [[LOAD3]](<2 x s64>)
; GCN-NEXT: [[UV:%[0-9]+]]:vgpr(s64), [[UV1:%[0-9]+]]:vgpr(s64), [[UV2:%[0-9]+]]:vgpr(s64), [[UV3:%[0-9]+]]:vgpr(s64), [[UV4:%[0-9]+]]:vgpr(s64), [[UV5:%[0-9]+]]:vgpr(s64), [[UV6:%[0-9]+]]:vgpr(s64), [[UV7:%[0-9]+]]:vgpr(s64) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x s64>)
@@ -889,8 +889,8 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr(p4) = COPY $vgpr0_vgpr1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[COPY]](p4) :: (load (<4 x s32>) from constant-pool, align 32, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:vgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:vgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p4) :: (load (<4 x s32>) from constant-pool + 16, basealign 32, addrspace 4)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
%0:_(p4) = COPY $vgpr0_vgpr1
@@ -917,8 +917,8 @@ body: |
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[PHI:%[0-9]+]]:vgpr(p4) = G_PHI [[COPY]](p4), %bb.0, %3(p4), %bb.1
; GCN-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PHI]](p4) :: (load (<4 x s32>) from constant-pool, align 32, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:vgpr(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p4) = nuw inbounds G_PTR_ADD [[PHI]], [[C]](s64)
+ ; GCN-NEXT: [[C:%[0-9]+]]:vgpr(i64) = G_CONSTANT i64 16
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p4) = nuw inbounds G_PTR_ADD [[PHI]], [[C]](i64)
; GCN-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p4) :: (load (<4 x s32>) from constant-pool + 16, basealign 32, addrspace 4)
; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr(p4) = COPY [[COPY1]](p4)
@@ -951,8 +951,8 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(<2 x s32>) = G_LOAD [[COPY]](p4) :: (invariant load (<2 x s32>) from constant-pool, align 4, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(s32) = G_LOAD [[PTR_ADD]](p4) :: (invariant load (s32) from constant-pool + 8, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s32>)
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<3 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[LOAD1]](s32)
@@ -982,8 +982,8 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(<2 x s32>) = G_LOAD [[COPY]](p4) :: (invariant load (<2 x s32>) from constant-pool, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(s32) = G_LOAD [[PTR_ADD]](p4) :: (invariant load (s32) from constant-pool + 8, align 8, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s32>)
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<3 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[LOAD1]](s32)
@@ -1041,8 +1041,8 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(<4 x s16>) = G_LOAD [[COPY]](p4) :: (invariant load (<4 x s16>) from constant-pool, align 4, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(<2 x s16>) = G_LOAD [[PTR_ADD]](p4) :: (invariant load (<2 x s16>) from constant-pool + 8, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(<2 x s16>), [[UV1:%[0-9]+]]:sgpr(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:sgpr(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[LOAD1]](<2 x s16>)
@@ -1072,8 +1072,8 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(<4 x s16>) = G_LOAD [[COPY]](p4) :: (invariant load (<4 x s16>) from constant-pool, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(<2 x s16>) = G_LOAD [[PTR_ADD]](p4) :: (invariant load (<2 x s16>) from constant-pool + 8, align 8, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(<2 x s16>), [[UV1:%[0-9]+]]:sgpr(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:sgpr(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[LOAD1]](<2 x s16>)
@@ -1131,8 +1131,8 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(s64) = G_LOAD [[COPY]](p4) :: (invariant load (s64) from constant-pool, align 4, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(s32) = G_LOAD [[PTR_ADD]](p4) :: (invariant load (s32) from constant-pool + 8, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[LOAD]](s64)
; GFX7-NEXT: [[MV:%[0-9]+]]:sgpr(s96) = G_MERGE_VALUES [[UV]](s32), [[UV1]](s32), [[LOAD1]](s32)
@@ -1162,8 +1162,8 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(s64) = G_LOAD [[COPY]](p4) :: (invariant load (s64) from constant-pool, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(s32) = G_LOAD [[PTR_ADD]](p4) :: (invariant load (s32) from constant-pool + 8, align 8, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[LOAD]](s64)
; GFX7-NEXT: [[MV:%[0-9]+]]:sgpr(s96) = G_MERGE_VALUES [[UV]](s32), [[UV1]](s32), [[LOAD1]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
index 60b89bf42031d..44478f7d7f205 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
@@ -97,10 +97,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[ZEXT]], [[ZEXT1]](s32)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[ZEXT]], [[ZEXT1]](i32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC2]](s16)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -199,17 +199,17 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[BITCAST]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[BITCAST1]], [[C]]
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:sgpr(s32) = G_LSHR [[AND]], [[AND1]](s32)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:sgpr(s32) = G_LSHR [[LSHR]], [[LSHR1]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[LSHR2]](s32), [[LSHR3]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[BITCAST]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[BITCAST1]], [[C]]
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:sgpr(i32) = G_LSHR [[AND]], [[AND1]](i32)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:sgpr(i32) = G_LSHR [[LSHR]], [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[LSHR2]](i32), [[LSHR3]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR_TRUNC]](<2 x s16>)
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
index 3674fb9156f7a..e3853281c6762 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbanklegalize.mir
@@ -226,9 +226,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ABS:%[0-9]+]]:sgpr(s32) = G_ABS [[SEXT]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ABS]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:sgpr(i32) = G_ABS [[SEXT]]
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ABS]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s16)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ANYEXT]](s32)
; CHECK-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s16), addrspace 1)
@@ -260,27 +260,27 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP1]], [[C2]]
- ; CHECK-NEXT: G_BRCOND [[AND]](s32), %bb.2
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP1]], [[C2]]
+ ; CHECK-NEXT: G_BRCOND [[AND]](i32), %bb.2
; CHECK-NEXT: G_BR %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C3]]
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C3]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI [[ICMP]](s32), %bb.0, [[ICMP2]](s32), %bb.1
- ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[PHI]], [[C4]]
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:sgpr(i32) = G_PHI [[ICMP]](i32), %bb.0, [[ICMP2]](i32), %bb.1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[PHI]], [[C4]]
; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](s32), [[C5]], [[C6]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](i32), [[C5]], [[C6]]
; CHECK-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; CHECK-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[SELECT]], [[C7]]
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
@@ -337,10 +337,10 @@ body: |
; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(oeq), [[COPY5]](s32), [[COPY6]]
- ; CHECK-NEXT: [[AMDGPU_COPY_SCC_VCC:%[0-9]+]]:sgpr(s32) = G_AMDGPU_COPY_SCC_VCC [[FCMP]](s1)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_COPY_SCC_VCC]], [[C1]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[COPY1]], [[COPY2]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_SCC_VCC:%[0-9]+]]:sgpr(i32) = G_AMDGPU_COPY_SCC_VCC [[FCMP]](s1)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[AMDGPU_COPY_SCC_VCC]], [[C1]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[COPY1]], [[COPY2]]
; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; CHECK-NEXT: G_STORE [[COPY7]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
@@ -374,8 +374,8 @@ body: |
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[COPY1]], [[COPY2]]
; CHECK-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
@@ -442,11 +442,12 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C2]], [[C3]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; CHECK-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
@@ -476,12 +477,12 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C2]], [[C3]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; CHECK-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
@@ -552,13 +553,13 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 20
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[AND]], [[C2]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](s32), [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[ICMP1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[AND]], [[C2]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](i32), [[COPY]], [[COPY1]]
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; CHECK-NEXT: G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
@@ -742,8 +743,8 @@ body: |
; CHECK-NEXT: [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
; CHECK-NEXT: [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
; CHECK-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI2]], [[C1]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI2]], [[C1]](i32)
; CHECK-NEXT: [[MV3:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[PHI2]](s32), [[ASHR]](s32)
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C2]](s32)
@@ -809,7 +810,7 @@ body: |
; CHECK-NEXT: G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
; CHECK-NEXT: [[ADD1:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI2]], [[C9]]
; CHECK-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
- ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ult), [[PHI2]](s32), [[C10]]
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[PHI2]](s32), [[C10]]
; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP2]], [[C9]]
; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC2:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[AND]](s32)
; CHECK-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY14]](s1), $exec_lo, implicit-def $scc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
index e06ac27ac2b40..091d7617198cf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
@@ -341,9 +341,9 @@ body: |
; NEW_RBS-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
- ; NEW_RBS-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; NEW_RBS-NEXT: [[ABS:%[0-9]+]]:sgpr(s32) = G_ABS [[SEXT]]
- ; NEW_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ABS]](s32)
+ ; NEW_RBS-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; NEW_RBS-NEXT: [[ABS:%[0-9]+]]:sgpr(i32) = G_ABS [[SEXT]]
+ ; NEW_RBS-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[ABS]](i32)
; NEW_RBS-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s16)
; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[ANYEXT]](s32)
; NEW_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s16), addrspace 1)
@@ -414,27 +414,27 @@ body: |
; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 6
- ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+ ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
- ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP1]], [[C2]]
- ; NEW_RBS-NEXT: G_BRCOND [[AND]](s32), %bb.2
+ ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+ ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP1]], [[C2]]
+ ; NEW_RBS-NEXT: G_BRCOND [[AND]](i32), %bb.2
; NEW_RBS-NEXT: G_BR %bb.1
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.1:
; NEW_RBS-NEXT: successors: %bb.2(0x80000000)
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[ICMP2:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C3]]
+ ; NEW_RBS-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C3]]
; NEW_RBS-NEXT: {{ $}}
; NEW_RBS-NEXT: bb.2:
- ; NEW_RBS-NEXT: [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI [[ICMP]](s32), %bb.0, [[ICMP2]](s32), %bb.1
- ; NEW_RBS-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[PHI]], [[C4]]
+ ; NEW_RBS-NEXT: [[PHI:%[0-9]+]]:sgpr(i32) = G_PHI [[ICMP]](i32), %bb.0, [[ICMP2]](i32), %bb.1
+ ; NEW_RBS-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[PHI]], [[C4]]
; NEW_RBS-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
; NEW_RBS-NEXT: [[C6:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](s32), [[C5]], [[C6]]
+ ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](i32), [[C5]], [[C6]]
; NEW_RBS-NEXT: [[C7:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; NEW_RBS-NEXT: [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[SELECT]], [[C7]]
; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
@@ -510,10 +510,10 @@ body: |
; NEW_RBS-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; NEW_RBS-NEXT: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(oeq), [[COPY5]](s32), [[COPY6]]
- ; NEW_RBS-NEXT: [[AMDGPU_COPY_SCC_VCC:%[0-9]+]]:sgpr(s32) = G_AMDGPU_COPY_SCC_VCC [[FCMP]](s1)
- ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_COPY_SCC_VCC]], [[C1]]
- ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[COPY1]], [[COPY2]]
+ ; NEW_RBS-NEXT: [[AMDGPU_COPY_SCC_VCC:%[0-9]+]]:sgpr(i32) = G_AMDGPU_COPY_SCC_VCC [[FCMP]](s1)
+ ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[AMDGPU_COPY_SCC_VCC]], [[C1]]
+ ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[COPY1]], [[COPY2]]
; NEW_RBS-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; NEW_RBS-NEXT: G_STORE [[COPY7]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: S_ENDPGM 0
@@ -564,8 +564,8 @@ body: |
; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
- ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[COPY1]], [[COPY2]]
; NEW_RBS-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: S_ENDPGM 0
@@ -662,11 +662,12 @@ body: |
; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
- ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
- ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
+ ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C2]], [[C3]]
; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; NEW_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: S_ENDPGM 0
@@ -711,12 +712,12 @@ body: |
; NEW_RBS-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
- ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
+ ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+ ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
; NEW_RBS-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C2]], [[C3]]
+ ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C2]], [[C3]]
; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; NEW_RBS-NEXT: G_STORE [[COPY3]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: S_ENDPGM 0
@@ -830,13 +831,13 @@ body: |
; NEW_RBS-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; NEW_RBS-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
; NEW_RBS-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
+ ; NEW_RBS-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(uge), [[COPY]](s32), [[C]]
; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 20
- ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
- ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[ICMP1]]
- ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; NEW_RBS-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[AND]], [[C2]]
- ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](s32), [[COPY]], [[COPY1]]
+ ; NEW_RBS-NEXT: [[ICMP1:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(uge), [[COPY1]](s32), [[C1]]
+ ; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[ICMP1]]
+ ; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; NEW_RBS-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[AND]], [[C2]]
+ ; NEW_RBS-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND1]](i32), [[COPY]], [[COPY1]]
; NEW_RBS-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[SELECT]](s32)
; NEW_RBS-NEXT: G_STORE [[COPY4]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: S_ENDPGM 0
@@ -1190,8 +1191,8 @@ body: |
; NEW_RBS-NEXT: [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %67(s32), %bb.3, [[C]](s32), %bb.0
; NEW_RBS-NEXT: [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
; NEW_RBS-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
- ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
- ; NEW_RBS-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI2]], [[C1]](s32)
+ ; NEW_RBS-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; NEW_RBS-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[PHI2]], [[C1]](i32)
; NEW_RBS-NEXT: [[MV3:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[PHI2]](s32), [[ASHR]](s32)
; NEW_RBS-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 2
; NEW_RBS-NEXT: [[SHL:%[0-9]+]]:sgpr(s64) = G_SHL [[MV3]], [[C2]](s32)
@@ -1257,7 +1258,7 @@ body: |
; NEW_RBS-NEXT: G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
; NEW_RBS-NEXT: [[ADD1:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI2]], [[C9]]
; NEW_RBS-NEXT: [[C10:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 100
- ; NEW_RBS-NEXT: [[ICMP2:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ult), [[PHI2]](s32), [[C10]]
+ ; NEW_RBS-NEXT: [[ICMP2:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ult), [[PHI2]](s32), [[C10]]
; NEW_RBS-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP2]], [[C9]]
; NEW_RBS-NEXT: [[AMDGPU_COPY_VCC_SCC2:%[0-9]+]]:sreg_32(s1) = G_AMDGPU_COPY_VCC_SCC [[AND]](s32)
; NEW_RBS-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY14]](s1), $exec_lo, implicit-def $scc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir
index 7a7353a3a168d..47fa6d9b960d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mul.mir
@@ -105,15 +105,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
- ; CHECK-NEXT: [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[UV]], [[UV2]]
- ; CHECK-NEXT: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[UV]], [[UV2]]
- ; CHECK-NEXT: [[MUL1:%[0-9]+]]:vgpr(s32) = G_MUL [[UV]], [[UV3]]
- ; CHECK-NEXT: [[MUL2:%[0-9]+]]:vgpr(s32) = G_MUL [[UV1]], [[UV2]]
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[MUL1]], [[MUL2]]
- ; CHECK-NEXT: [[ADD1:%[0-9]+]]:vgpr(s32) = G_ADD [[ADD]], [[UMULH]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[MUL]](s32), [[ADD1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY1]](s64)
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:vgpr(i32) = G_MUL [[UV]], [[UV2]]
+ ; CHECK-NEXT: [[UMULH:%[0-9]+]]:vgpr(i32) = G_UMULH [[UV]], [[UV2]]
+ ; CHECK-NEXT: [[MUL1:%[0-9]+]]:vgpr(i32) = G_MUL [[UV]], [[UV3]]
+ ; CHECK-NEXT: [[MUL2:%[0-9]+]]:vgpr(i32) = G_MUL [[UV1]], [[UV2]]
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(i32) = G_ADD [[MUL1]], [[MUL2]]
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:vgpr(i32) = G_ADD [[ADD]], [[UMULH]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[MUL]](i32), [[ADD1]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(s64) = G_MUL %0, %1
@@ -149,10 +149,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s32) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s32) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[AMDGPU_MAD_U64_U32_:%[0-9]+]]:vgpr(s64), [[AMDGPU_MAD_U64_U32_1:%[0-9]+]]:sgpr(s32) = G_AMDGPU_MAD_U64_U32 [[TRUNC]](s32), [[TRUNC1]], [[C]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:vgpr(i32) = G_TRUNC [[COPY]](s64)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(i32) = G_TRUNC [[COPY1]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[AMDGPU_MAD_U64_U32_:%[0-9]+]]:vgpr(s64), [[AMDGPU_MAD_U64_U32_1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_MAD_U64_U32 [[TRUNC]](i32), [[TRUNC1]], [[C]]
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(s64) = G_AMDGPU_S_MUL_U64_U32 %0, %1
@@ -188,10 +188,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s32) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(s32) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[AMDGPU_MAD_I64_I32_:%[0-9]+]]:vgpr(s64), [[AMDGPU_MAD_I64_I32_1:%[0-9]+]]:sgpr(s32) = G_AMDGPU_MAD_I64_I32 [[TRUNC]](s32), [[TRUNC1]], [[C]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:vgpr(i32) = G_TRUNC [[COPY]](s64)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:vgpr(i32) = G_TRUNC [[COPY1]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[AMDGPU_MAD_I64_I32_:%[0-9]+]]:vgpr(s64), [[AMDGPU_MAD_I64_I32_1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_MAD_I64_I32 [[TRUNC]](i32), [[TRUNC1]], [[C]]
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(s64) = G_AMDGPU_S_MUL_I64_I32 %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
index ae03b781db28d..2f54715ccb8d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
@@ -88,10 +88,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY1]](s32), [[C]]
- ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: S_NOP 0, implicit [[OR]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY1]](s32), [[C]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[ICMP]], [[ICMP1]]
+ ; CHECK-NEXT: S_NOP 0, implicit [[OR]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = G_CONSTANT i32 0
@@ -143,10 +143,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:vcc(s1) = G_OR [[AMDGPU_COPY_VCC_SCC]], [[ICMP1]]
; CHECK-NEXT: S_NOP 0, implicit [[OR]](s1)
%0:_(s32) = COPY $sgpr0
@@ -169,8 +169,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[COPY]], [[COPY1]]
- ; CHECK-NEXT: S_NOP 0, implicit [[OR]](s32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: S_NOP 0, implicit [[OR]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_TRUNC %0
@@ -192,9 +192,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[COPY]], [[ICMP]]
- ; CHECK-NEXT: S_NOP 0, implicit [[OR]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[COPY]], [[ICMP]]
+ ; CHECK-NEXT: S_NOP 0, implicit [[OR]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
index 3b78f1e3a2157..c663442495aab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
@@ -15,10 +15,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
- ; CHECK-NEXT: [[SADDE:%[0-9]+]]:sgpr(s32), [[SADDE1:%[0-9]+]]:sgpr(s32) = G_SADDE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
+ ; CHECK-NEXT: [[SADDE:%[0-9]+]]:sgpr(s32), [[SADDE1:%[0-9]+]]:sgpr(i32) = G_SADDE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
@@ -41,9 +41,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY]], [[COPY3]], [[AMDGPU_COPY_VCC_SCC]]
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $sgpr0
@@ -114,9 +114,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[SADDE:%[0-9]+]]:sgpr(s32), [[SADDE1:%[0-9]+]]:sgpr(s32) = G_SADDE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[SADDE:%[0-9]+]]:sgpr(s32), [[SADDE1:%[0-9]+]]:sgpr(i32) = G_SADDE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-salu-float.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-salu-float.mir
index a030c2a213342..a2223defb765a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-salu-float.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-salu-float.mir
@@ -205,9 +205,9 @@ body: |
; GFX1150-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; GFX1150-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[TRUNC]](s16)
; GFX1150-NEXT: [[FPTOUI:%[0-9]+]]:vgpr(s16) = G_FPTOUI [[COPY1]](s16)
- ; GFX1150-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[FPTOUI]](s16)
- ; GFX1150-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[ANYEXT]]
- ; GFX1150-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[AMDGPU_READANYLANE]](s32)
+ ; GFX1150-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[FPTOUI]](s16)
+ ; GFX1150-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[ANYEXT]]
+ ; GFX1150-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[AMDGPU_READANYLANE]](i32)
%1:_(s32) = COPY $sgpr0
%0:_(s16) = G_TRUNC %1(s32)
%2:_(s16) = G_FPTOUI %0(s16)
@@ -245,9 +245,9 @@ body: |
; GFX1150-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; GFX1150-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[TRUNC]](s16)
; GFX1150-NEXT: [[FPTOUI_SAT:%[0-9]+]]:vgpr(s16) = G_FPTOUI_SAT [[COPY1]](s16)
- ; GFX1150-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[FPTOUI_SAT]](s16)
- ; GFX1150-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[ANYEXT]]
- ; GFX1150-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[AMDGPU_READANYLANE]](s32)
+ ; GFX1150-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[FPTOUI_SAT]](s16)
+ ; GFX1150-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[ANYEXT]]
+ ; GFX1150-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[AMDGPU_READANYLANE]](i32)
%1:_(s32) = COPY $sgpr0
%0:_(s16) = G_TRUNC %1(s32)
%2:_(s16) = G_FPTOUI_SAT %0(s16)
@@ -288,9 +288,9 @@ body: |
; GFX1150-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; GFX1150-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[TRUNC]](s16)
; GFX1150-NEXT: [[UITOFP:%[0-9]+]]:vgpr(s16) = G_UITOFP [[COPY1]](s16)
- ; GFX1150-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[UITOFP]](s16)
- ; GFX1150-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[ANYEXT]]
- ; GFX1150-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[AMDGPU_READANYLANE]](s32)
+ ; GFX1150-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[UITOFP]](s16)
+ ; GFX1150-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[ANYEXT]]
+ ; GFX1150-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[AMDGPU_READANYLANE]](i32)
%1:_(s32) = COPY $sgpr0
%0:_(s16) = G_TRUNC %1(s32)
%2:_(s16) = G_UITOFP %0(s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir
index 572f1ea2516f1..29fc51ee4c41d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sbfx.mir
@@ -94,11 +94,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s64) = G_ASHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 64
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(s32) = G_SUB [[C]], [[COPY2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[ASHR]], [[SUB]](s32)
- ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(s64) = G_ASHR [[SHL]], [[SUB]](s32)
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i64) = G_ASHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(i32) = G_SUB [[C]], [[COPY2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(i64) = G_SHL [[ASHR]], [[SUB]](i32)
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(s64) = G_ASHR [[SHL]], [[SUB]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[ASHR1]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = COPY $vgpr2
@@ -121,11 +121,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s64) = G_ASHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 64
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(s32) = G_SUB [[C]], [[COPY2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[ASHR]], [[SUB]](s32)
- ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(s64) = G_ASHR [[SHL]], [[SUB]](s32)
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i64) = G_ASHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(i32) = G_SUB [[C]], [[COPY2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(i64) = G_SHL [[ASHR]], [[SUB]](i32)
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(s64) = G_ASHR [[SHL]], [[SUB]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[ASHR1]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = COPY $vgpr0
@@ -152,13 +152,13 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s64) = G_ASHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SBFX:%[0-9]+]]:vgpr(s32) = G_SBFX [[UV]], [[C2]](s32), [[COPY2]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(s32) = G_ASHR [[SBFX]], [[C3]](s32)
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SBFX]](s32), [[ASHR1]](s32)
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i64) = G_ASHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[ASHR]](i64)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SBFX:%[0-9]+]]:vgpr(i32) = G_SBFX [[UV]], [[C2]](i32), [[COPY2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(i32) = G_ASHR [[SBFX]], [[C3]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SBFX]](i32), [[ASHR1]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 31
@@ -183,12 +183,12 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 40
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s64) = G_ASHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SBFX:%[0-9]+]]:vgpr(s32) = G_SBFX [[UV1]], [[C2]](s32), [[C3]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](s32), [[SBFX]](s32)
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i64) = G_ASHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[ASHR]](i64)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SBFX:%[0-9]+]]:vgpr(i32) = G_SBFX [[UV1]], [[C2]](i32), [[C3]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](i32), [[SBFX]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 8
@@ -212,11 +212,11 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY [[COPY]](s64)
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s64) = G_ASHR [[COPY3]], [[COPY1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 64
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(s32) = G_SUB [[C]], [[COPY2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[ASHR]], [[SUB]](s32)
- ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(s64) = G_ASHR [[SHL]], [[SUB]](s32)
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i64) = G_ASHR [[COPY3]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(i32) = G_SUB [[C]], [[COPY2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(i64) = G_SHL [[ASHR]], [[SUB]](i32)
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:vgpr(s64) = G_ASHR [[SHL]], [[SUB]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[ASHR1]](s64)
%0:_(s64) = COPY $sgpr0_sgpr1
%1:_(s32) = COPY $vgpr0
@@ -265,14 +265,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY2]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY2]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32(s32) = COPY [[COPY]](s32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s32) = COPY [[OR]](s32)
- ; CHECK-NEXT: [[S_BFE_I32_:%[0-9]+]]:sreg_32(s32) = S_BFE_I32 [[COPY3]](s32), [[COPY4]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY [[OR]](i32)
+ ; CHECK-NEXT: [[S_BFE_I32_:%[0-9]+]]:sreg_32(s32) = S_BFE_I32 [[COPY3]](s32), [[COPY4]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY5:%[0-9]+]]:sgpr(s32) = COPY [[S_BFE_I32_]](s32)
; CHECK-NEXT: $sgpr0 = COPY [[COPY5]](s32)
%0:_(s32) = COPY $sgpr0
@@ -296,13 +296,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655360
- ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655361
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655360
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655361
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32(s32) = COPY [[COPY]](s32)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(s32) = COPY [[C5]](s32)
- ; CHECK-NEXT: [[S_BFE_I32_:%[0-9]+]]:sreg_32(s32) = S_BFE_I32 [[COPY1]](s32), [[COPY2]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY [[C6]](i32)
+ ; CHECK-NEXT: [[S_BFE_I32_:%[0-9]+]]:sreg_32(s32) = S_BFE_I32 [[COPY1]](s32), [[COPY2]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY [[S_BFE_I32_]](s32)
; CHECK-NEXT: $sgpr0 = COPY [[COPY3]](s32)
%0:_(s32) = COPY $sgpr0
@@ -328,14 +329,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY2]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY2]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_64(s64) = COPY [[COPY]](s64)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s32) = COPY [[OR]](s32)
- ; CHECK-NEXT: [[S_BFE_I64_:%[0-9]+]]:sreg_64(s64) = S_BFE_I64 [[COPY3]](s64), [[COPY4]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY [[OR]](i32)
+ ; CHECK-NEXT: [[S_BFE_I64_:%[0-9]+]]:sreg_64(s64) = S_BFE_I64 [[COPY3]](s64), [[COPY4]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY5:%[0-9]+]]:sgpr(s64) = COPY [[S_BFE_I64_]](s64)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[COPY5]](s64)
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -359,13 +360,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655360
- ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655361
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655360
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655361
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_64(s64) = COPY [[COPY]](s64)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(s32) = COPY [[C5]](s32)
- ; CHECK-NEXT: [[S_BFE_I64_:%[0-9]+]]:sreg_64(s64) = S_BFE_I64 [[COPY1]](s64), [[COPY2]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY [[C6]](i32)
+ ; CHECK-NEXT: [[S_BFE_I64_:%[0-9]+]]:sreg_64(s64) = S_BFE_I64 [[COPY1]](s64), [[COPY2]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s64) = COPY [[S_BFE_I64_]](s64)
%0:_(s64) = COPY $sgpr0_sgpr1
%1:_(s32) = G_CONSTANT i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
index 2fd2e03471f1d..183f15eacf012 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
@@ -14,10 +14,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
@@ -39,8 +39,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[COPY4]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
@@ -65,8 +65,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[COPY3]], [[COPY4]]
%0:_(s32) = COPY $sgpr0
@@ -90,8 +90,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -210,10 +210,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s64) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](i32), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s64) = COPY $sgpr2_sgpr3
@@ -235,8 +235,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s64) = COPY [[COPY2]](s64)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](s64)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](s64)
@@ -265,8 +265,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s64) = COPY [[COPY2]](s64)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](s64)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](s64)
@@ -294,8 +294,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY2]](s64)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](s64)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV]], [[UV2]]
@@ -461,10 +461,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(<2 x s32>) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(<2 x s32>) = G_SELECT [[AND]](i32), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(<2 x s32>) = COPY $sgpr2_sgpr3
@@ -486,8 +486,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s32>) = COPY [[COPY2]](<2 x s32>)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](<2 x s32>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](<2 x s32>)
@@ -516,8 +516,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s32>) = COPY [[COPY2]](<2 x s32>)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](<2 x s32>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](<2 x s32>)
@@ -545,8 +545,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](<2 x s32>)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV]], [[UV2]]
@@ -685,10 +685,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(<4 x s16>) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(<4 x s16>) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(<4 x s16>) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(<4 x s16>) = G_SELECT [[AND]](i32), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(<4 x s16>) = COPY $sgpr2_sgpr3
@@ -710,8 +710,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(<4 x s16>) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY2]](<4 x s16>)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x s16>), [[UV1:%[0-9]+]]:vgpr(<2 x s16>) = G_UNMERGE_VALUES [[COPY4]](<4 x s16>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(<2 x s16>), [[UV3:%[0-9]+]]:vgpr(<2 x s16>) = G_UNMERGE_VALUES [[COPY3]](<4 x s16>)
@@ -740,8 +740,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(<4 x s16>) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY2]](<4 x s16>)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x s16>), [[UV1:%[0-9]+]]:vgpr(<2 x s16>) = G_UNMERGE_VALUES [[COPY3]](<4 x s16>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(<2 x s16>), [[UV3:%[0-9]+]]:vgpr(<2 x s16>) = G_UNMERGE_VALUES [[COPY4]](<4 x s16>)
@@ -769,8 +769,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(<2 x s16>), [[UV1:%[0-9]+]]:vgpr(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(<2 x s16>), [[UV3:%[0-9]+]]:vgpr(<2 x s16>) = G_UNMERGE_VALUES [[COPY3]](<4 x s16>)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV]], [[UV2]]
@@ -909,10 +909,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(p0) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(p0) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(p0) = G_SELECT [[AND]](i32), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(p0) = COPY $sgpr2_sgpr3
@@ -934,10 +934,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p1) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(p1) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(p1) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(p1) = G_SELECT [[AND]](i32), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(p1) = COPY $sgpr2_sgpr3
@@ -959,10 +959,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p999) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(p999) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(p999) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(p999) = G_SELECT [[AND]](i32), [[COPY2]], [[COPY3]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(p999) = COPY $sgpr2_sgpr3
@@ -984,8 +984,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p0) = COPY [[COPY2]](p0)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p0)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p0)
@@ -1014,8 +1014,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p1) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p1) = COPY [[COPY2]](p1)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p1)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p1)
@@ -1044,8 +1044,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p0) = COPY [[COPY2]](p0)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p0)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p0)
@@ -1073,8 +1073,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p1) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p1) = COPY [[COPY2]](p1)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p1)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p1)
@@ -1102,8 +1102,8 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p1) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY2]](p1)
; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p1)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV]], [[UV2]]
@@ -1496,9 +1496,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[COPY1]], [[COPY2]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[COPY1]], [[COPY2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir
index 1a8fa56a7f799..851a17e7092b7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext-inreg.mir
@@ -133,12 +133,12 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:vgpr(s32) = G_FREEZE [[UV]]
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(s32) = G_SEXT_INREG [[FREEZE]], 1
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[SEXT_INREG]], [[C]](s32)
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SEXT_INREG]](s32), [[ASHR]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:vgpr(i32) = G_FREEZE [[UV]]
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(i32) = G_SEXT_INREG [[FREEZE]], 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i32) = G_ASHR [[SEXT_INREG]], [[C]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SEXT_INREG]](i32), [[ASHR]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_SEXT_INREG %0, 1
@@ -158,12 +158,12 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:vgpr(s32) = G_FREEZE [[UV]]
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(s32) = G_SEXT_INREG [[FREEZE]], 31
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[SEXT_INREG]], [[C]](s32)
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SEXT_INREG]](s32), [[ASHR]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:vgpr(i32) = G_FREEZE [[UV]]
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(i32) = G_SEXT_INREG [[FREEZE]], 31
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i32) = G_ASHR [[SEXT_INREG]], [[C]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SEXT_INREG]](i32), [[ASHR]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_SEXT_INREG %0, 31
@@ -183,11 +183,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:vgpr(s32) = G_FREEZE [[UV]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[FREEZE]], [[C]](s32)
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[FREEZE]](s32), [[ASHR]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:vgpr(i32) = G_FREEZE [[UV]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i32) = G_ASHR [[FREEZE]], [[C]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[FREEZE]](i32), [[ASHR]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_SEXT_INREG %0, 32
@@ -207,9 +207,9 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(s32) = G_SEXT_INREG [[UV1]], 1
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](s32), [[SEXT_INREG]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(i32) = G_SEXT_INREG [[UV1]], 1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](i32), [[SEXT_INREG]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_SEXT_INREG %0, 33
@@ -229,9 +229,9 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(s32) = G_SEXT_INREG [[UV1]], 3
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](s32), [[SEXT_INREG]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(i32) = G_SEXT_INREG [[UV1]], 3
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](i32), [[SEXT_INREG]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_SEXT_INREG %0, 35
@@ -251,9 +251,9 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(s32) = G_SEXT_INREG [[UV1]], 31
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](s32), [[SEXT_INREG]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:vgpr(i32) = G_SEXT_INREG [[UV1]], 31
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](i32), [[SEXT_INREG]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_SEXT_INREG %0, 63
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir
index e0e783e7a62f9..fd489ae9e6902 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sext.mir
@@ -12,8 +12,8 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[COPY]], [[C]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[COPY]], [[C]](i32)
; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[ASHR]](s32)
%0:_(s32) = COPY $sgpr0
%1:_(s64) = G_SEXT %0
@@ -31,10 +31,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[SEXT]], [[C]](s32)
- ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[SEXT]](s32), [[ASHR]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(i32) = G_ASHR [[SEXT]], [[C]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[SEXT]](i32), [[ASHR]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s16) = G_TRUNC %0
%2:_(s64) = G_SEXT %1
@@ -51,8 +51,8 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[COPY]], [[C]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[COPY]], [[C]](i32)
; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[ASHR]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s64) = G_SEXT %0
@@ -70,13 +70,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(i32) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](i32)
; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s16) = G_CONSTANT i16 255
; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s16) = G_AND [[TRUNC]], [[C3]]
%0:_(s32) = COPY $sgpr0
@@ -99,12 +99,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -123,12 +123,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 -1
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -192,10 +192,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](i32), [[SELECT]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -213,12 +213,12 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(i32) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](i32)
; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s16) = G_CONSTANT i16 255
; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s16) = G_AND [[TRUNC]], [[C3]]
%0:_(s32) = COPY $sgpr0
@@ -239,11 +239,11 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s1) = G_TRUNC %0
%2:_(s32) = G_SEXT %1
@@ -260,11 +260,11 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 -1
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s1) = G_TRUNC %0
%2:_(s64) = G_SEXT %1
@@ -330,9 +330,10 @@ body: |
; CHECK-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[C2]], [[C1]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](i32), [[SELECT]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s1) = G_TRUNC %0
%2:_(s64) = G_SEXT %1
@@ -350,10 +351,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:vgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[SEXT]], [[C]](s32)
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SEXT]](s32), [[ASHR]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:vgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:vgpr(i32) = G_ASHR [[SEXT]], [[C]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SEXT]](i32), [[ASHR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s16) = G_TRUNC %0
%2:_(s64) = G_SEXT %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir
index 6bdf8e7e1de6f..4a88d5a2a7848 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-shl.mir
@@ -97,10 +97,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[ANYEXT]], [[ZEXT]](s32)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SHL]](s32)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[ANYEXT]], [[ZEXT]](i32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SHL]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC2]](s16)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -199,14 +199,14 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[BITCAST]], [[BITCAST1]](s32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:sgpr(s32) = G_SHL [[LSHR]], [[LSHR1]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SHL]](s32), [[SHL1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[BITCAST]], [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:sgpr(i32) = G_SHL [[LSHR]], [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SHL]](i32), [[SHL1]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR_TRUNC]](<2 x s16>)
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir
index a7023d0ecb507..a557a72552456 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smax.mir
@@ -121,10 +121,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[SMAX:%[0-9]+]]:sgpr(s32) = G_SMAX [[SEXT]], [[SEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMAX]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:sgpr(i32) = G_SMAX [[SEXT]], [[SEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMAX]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -152,10 +152,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[SMAX:%[0-9]+]]:sgpr(s32) = G_SMAX [[SEXT]], [[SEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMAX]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:sgpr(i32) = G_SMAX [[SEXT]], [[SEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMAX]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -181,16 +181,16 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[BITCAST]], 16
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[BITCAST1]], 16
- ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:sgpr(s32) = G_ASHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[SMAX:%[0-9]+]]:sgpr(s32) = G_SMAX [[SEXT_INREG]], [[SEXT_INREG1]]
- ; CHECK-NEXT: [[SMAX1:%[0-9]+]]:sgpr(s32) = G_SMAX [[ASHR]], [[ASHR1]]
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SMAX]](s32), [[SMAX1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(i32) = G_SEXT_INREG [[BITCAST]], 16
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(i32) = G_ASHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:sgpr(i32) = G_SEXT_INREG [[BITCAST1]], 16
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:sgpr(i32) = G_ASHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:sgpr(i32) = G_SMAX [[SEXT_INREG]], [[SEXT_INREG1]]
+ ; CHECK-NEXT: [[SMAX1:%[0-9]+]]:sgpr(i32) = G_SMAX [[ASHR]], [[ASHR1]]
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SMAX]](i32), [[SMAX1]](i32)
; CHECK-NEXT: $sgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir
index 9dd5f45d26077..abf0b48a133a5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smin.mir
@@ -124,10 +124,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[SMIN:%[0-9]+]]:sgpr(s32) = G_SMIN [[SEXT]], [[SEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMIN]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:sgpr(i32) = G_SMIN [[SEXT]], [[SEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMIN]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -155,10 +155,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(s32) = G_SEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[SMIN:%[0-9]+]]:sgpr(s32) = G_SMIN [[SEXT]], [[SEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMIN]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:sgpr(i32) = G_SEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:sgpr(i32) = G_SMIN [[SEXT]], [[SEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SMIN]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -184,16 +184,16 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[BITCAST]], 16
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[BITCAST1]], 16
- ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:sgpr(s32) = G_ASHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[SMIN:%[0-9]+]]:sgpr(s32) = G_SMIN [[SEXT_INREG]], [[SEXT_INREG1]]
- ; CHECK-NEXT: [[SMIN1:%[0-9]+]]:sgpr(s32) = G_SMIN [[ASHR]], [[ASHR1]]
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SMIN]](s32), [[SMIN1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(i32) = G_SEXT_INREG [[BITCAST]], 16
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:sgpr(i32) = G_ASHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:sgpr(i32) = G_SEXT_INREG [[BITCAST1]], 16
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:sgpr(i32) = G_ASHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:sgpr(i32) = G_SMIN [[SEXT_INREG]], [[SEXT_INREG1]]
+ ; CHECK-NEXT: [[SMIN1:%[0-9]+]]:sgpr(i32) = G_SMIN [[ASHR]], [[ASHR1]]
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SMIN]](i32), [[SMIN1]](i32)
; CHECK-NEXT: $sgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
index cdc673ea6804f..c6e232b720a4c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
@@ -36,19 +36,12 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>) from constant-pool, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(s32) = G_LOAD [[PTR_ADD]](p4) :: (load (s32) from constant-pool + 8, align 8, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s32>)
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<3 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[LOAD1]](s32)
; GFX7-NEXT: $sgpr0_sgpr1_sgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
- ;
- ; GFX12-LABEL: name: split_smrd_load_range
- ; GFX12: liveins: $sgpr0_sgpr1
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
- ; GFX12-NEXT: [[LOAD:%[0-9]+]]:sgpr(<3 x s32>) = G_LOAD [[COPY]](p4) :: (load (<3 x s32>) from constant-pool, align 8, !range {{.+}}, addrspace 4)
- ; GFX12-NEXT: $sgpr0_sgpr1_sgpr2 = COPY [[LOAD]](<3 x s32>)
%0:_(p4) = COPY $sgpr0_sgpr1
%1:_(<3 x s32>) = G_LOAD %0 :: (load (<3 x s32>) from constant-pool, align 8, addrspace 4, !range !3)
$sgpr0_sgpr1_sgpr2 = COPY %1
@@ -67,8 +60,8 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: [[LOAD:%[0-9]+]]:sgpr(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>) from constant-pool, !tbaa !2, addrspace 4)
- ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 8
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 8
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:sgpr(s32) = G_LOAD [[PTR_ADD]](p4) :: (load (s32) from constant-pool + 8, align 8, !tbaa !2, addrspace 4)
; GFX7-NEXT: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s32>)
; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<3 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[LOAD1]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
index e7b30219a1826..dfd99a4cd93ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
@@ -15,10 +15,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
- ; CHECK-NEXT: [[SSUBE:%[0-9]+]]:sgpr(s32), [[SSUBE1:%[0-9]+]]:sgpr(s32) = G_SSUBE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
+ ; CHECK-NEXT: [[SSUBE:%[0-9]+]]:sgpr(s32), [[SSUBE1:%[0-9]+]]:sgpr(i32) = G_SSUBE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
@@ -41,9 +41,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY]], [[COPY3]], [[AMDGPU_COPY_VCC_SCC]]
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $sgpr0
@@ -114,9 +114,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[SSUBE:%[0-9]+]]:sgpr(s32), [[SSUBE1:%[0-9]+]]:sgpr(s32) = G_SSUBE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[SSUBE:%[0-9]+]]:sgpr(s32), [[SSUBE1:%[0-9]+]]:sgpr(i32) = G_SSUBE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
index b820705785e11..855dcfa16ee52 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
@@ -15,10 +15,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:sgpr(s32) = G_SUB [[ANYEXT]], [[ANYEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SUB]](s32)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:sgpr(i32) = G_SUB [[ANYEXT]], [[ANYEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SUB]](i32)
; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s16) = G_AND [[TRUNC2]], [[TRUNC2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -196,14 +196,14 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:sgpr(s32) = G_SUB [[BITCAST]], [[BITCAST1]]
- ; CHECK-NEXT: [[SUB1:%[0-9]+]]:sgpr(s32) = G_SUB [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SUB]](s32), [[SUB1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:sgpr(i32) = G_SUB [[BITCAST]], [[BITCAST1]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:sgpr(i32) = G_SUB [[LSHR]], [[LSHR1]]
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[SUB]](i32), [[SUB1]](i32)
; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(<2 x s16>) = G_AND [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC]]
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
@@ -379,7 +379,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[USUBO:%[0-9]+]]:sgpr(s32), [[USUBO1:%[0-9]+]]:sgpr(s32) = G_USUBO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[USUBO:%[0-9]+]]:sgpr(s32), [[USUBO1:%[0-9]+]]:sgpr(i32) = G_USUBO [[COPY]], [[COPY1]]
; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[USUBO]], [[USUBO]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
@@ -462,9 +462,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:sgpr(s32) = G_USUBE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:sgpr(i32) = G_USUBE [[COPY]], [[COPY1]], [[AND]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[USUBE]], [[USUBE]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
index 03602ae3a443a..1c8f2da1edb59 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
@@ -62,12 +62,12 @@ body: |
; GFX: liveins: $vgpr0_vgpr1
; GFX-NEXT: {{ $}}
; GFX-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GFX-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; GFX-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
- ; GFX-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[C]]
- ; GFX-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[C1]]
- ; GFX-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+ ; GFX-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; GFX-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 1
+ ; GFX-NEXT: [[AND:%[0-9]+]]:vgpr(i32) = G_AND [[UV]], [[C]]
+ ; GFX-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX-NEXT: [[AND1:%[0-9]+]]:vgpr(i32) = G_AND [[UV1]], [[C1]]
+ ; GFX-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](i32), [[AND1]](i32)
; GFX-NEXT: [[C2:%[0-9]+]]:vgpr(s64) = G_CONSTANT i64 0
; GFX-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[MV]](s64), [[C2]]
%0:_(s64) = COPY $vgpr0_vgpr1
@@ -123,11 +123,11 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; GFX6-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
- ; GFX6-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[TRUNC]](s16)
- ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
- ; GFX6-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[ANYEXT]], [[C]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
+ ; GFX6-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(i32) = G_ANYEXT [[TRUNC]](s16)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 1
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:vgpr(i32) = G_AND [[ANYEXT]], [[C]]
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](i32), [[C1]]
;
; GFX8-LABEL: name: trunc_i16_to_i1_v
; GFX8: liveins: $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
index 52111500248ae..aef6308659341 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
@@ -14,10 +14,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
- ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(s32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(i32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
@@ -40,9 +40,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY]], [[COPY3]], [[AMDGPU_COPY_VCC_SCC]]
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $sgpr0
@@ -113,9 +113,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(s32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:sgpr(i32) = G_UADDE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
index 2fb0308f1e802..7f66bfb5b59a4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
@@ -13,7 +13,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[UADDO:%[0-9]+]]:sgpr(s32), [[UADDO1:%[0-9]+]]:sgpr(s32) = G_UADDO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:sgpr(s32), [[UADDO1:%[0-9]+]]:sgpr(i32) = G_UADDO [[COPY]], [[COPY1]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32), %3:_(s1) = G_UADDO %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir
index 267960ad74eff..a0060cb5d8a74 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ubfx.mir
@@ -94,11 +94,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(s64) = G_LSHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 64
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(s32) = G_SUB [[C]], [[COPY2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[LSHR]], [[SUB]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:vgpr(s64) = G_LSHR [[SHL]], [[SUB]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(i64) = G_LSHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(i32) = G_SUB [[C]], [[COPY2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(i64) = G_SHL [[LSHR]], [[SUB]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:vgpr(s64) = G_LSHR [[SHL]], [[SUB]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[LSHR1]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = COPY $vgpr2
@@ -121,11 +121,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(s64) = G_LSHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 64
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(s32) = G_SUB [[C]], [[COPY2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[LSHR]], [[SUB]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:vgpr(s64) = G_LSHR [[SHL]], [[SUB]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(i64) = G_LSHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(i32) = G_SUB [[C]], [[COPY2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(i64) = G_SHL [[LSHR]], [[SUB]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:vgpr(s64) = G_LSHR [[SHL]], [[SUB]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[LSHR1]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = COPY $vgpr0
@@ -152,11 +152,11 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 4
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(s64) = G_LSHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[LSHR]](s64)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[UBFX:%[0-9]+]]:vgpr(s32) = G_UBFX [[UV]], [[C2]](s32), [[COPY2]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UBFX]](s32), [[C2]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(i64) = G_LSHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[LSHR]](i64)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[UBFX:%[0-9]+]]:vgpr(i32) = G_UBFX [[UV]], [[C2]](i32), [[COPY2]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UBFX]](i32), [[C2]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 31
@@ -181,12 +181,12 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 40
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(s64) = G_LSHR [[COPY]], [[COPY1]](s32)
- ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[LSHR]](s64)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[UBFX:%[0-9]+]]:vgpr(s32) = G_UBFX [[UV1]], [[C2]](s32), [[C3]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](s32), [[UBFX]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(i64) = G_LSHR [[COPY]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[LSHR]](i64)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[UBFX:%[0-9]+]]:vgpr(i32) = G_UBFX [[UV1]], [[C2]](i32), [[C3]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[UV]](i32), [[UBFX]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 8
@@ -210,11 +210,11 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY [[COPY]](s64)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(s64) = G_LSHR [[COPY3]], [[COPY1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 64
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(s32) = G_SUB [[C]], [[COPY2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(s64) = G_SHL [[LSHR]], [[SUB]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:vgpr(s64) = G_LSHR [[SHL]], [[SUB]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:vgpr(i64) = G_LSHR [[COPY3]], [[COPY1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:vgpr(i32) = G_SUB [[C]], [[COPY2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:vgpr(i64) = G_SHL [[LSHR]], [[SUB]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:vgpr(s64) = G_LSHR [[SHL]], [[SUB]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[LSHR1]](s64)
%0:_(s64) = COPY $sgpr0_sgpr1
%1:_(s32) = COPY $vgpr0
@@ -263,14 +263,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY2]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY2]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32(s32) = COPY [[COPY]](s32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s32) = COPY [[OR]](s32)
- ; CHECK-NEXT: [[S_BFE_U32_:%[0-9]+]]:sreg_32(s32) = S_BFE_U32 [[COPY3]](s32), [[COPY4]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY [[OR]](i32)
+ ; CHECK-NEXT: [[S_BFE_U32_:%[0-9]+]]:sreg_32(s32) = S_BFE_U32 [[COPY3]](s32), [[COPY4]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY5:%[0-9]+]]:sgpr(s32) = COPY [[S_BFE_U32_]](s32)
; CHECK-NEXT: $sgpr0 = COPY [[COPY5]](s32)
%0:_(s32) = COPY $sgpr0
@@ -294,13 +294,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655360
- ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655361
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655360
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655361
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32(s32) = COPY [[COPY]](s32)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(s32) = COPY [[C5]](s32)
- ; CHECK-NEXT: [[S_BFE_U32_:%[0-9]+]]:sreg_32(s32) = S_BFE_U32 [[COPY1]](s32), [[COPY2]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY [[C6]](i32)
+ ; CHECK-NEXT: [[S_BFE_U32_:%[0-9]+]]:sreg_32(s32) = S_BFE_U32 [[COPY1]](s32), [[COPY2]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY [[S_BFE_U32_]](s32)
; CHECK-NEXT: $sgpr0 = COPY [[COPY3]](s32)
%0:_(s32) = COPY $sgpr0
@@ -326,14 +327,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[COPY2]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:sgpr(i32) = G_SHL [[COPY2]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_64(s64) = COPY [[COPY]](s64)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s32) = COPY [[OR]](s32)
- ; CHECK-NEXT: [[S_BFE_U64_:%[0-9]+]]:sreg_64(s64) = S_BFE_U64 [[COPY3]](s64), [[COPY4]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY [[OR]](i32)
+ ; CHECK-NEXT: [[S_BFE_U64_:%[0-9]+]]:sreg_64(s64) = S_BFE_U64 [[COPY3]](s64), [[COPY4]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY5:%[0-9]+]]:sgpr(s64) = COPY [[S_BFE_U64_]](s64)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[COPY5]](s64)
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -357,13 +358,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 63
- ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655360
- ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 655361
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655360
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 655361
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_64(s64) = COPY [[COPY]](s64)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(s32) = COPY [[C5]](s32)
- ; CHECK-NEXT: [[S_BFE_U64_:%[0-9]+]]:sreg_64(s64) = S_BFE_U64 [[COPY1]](s64), [[COPY2]](s32), implicit-def $scc
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY [[C6]](i32)
+ ; CHECK-NEXT: [[S_BFE_U64_:%[0-9]+]]:sreg_64(s64) = S_BFE_U64 [[COPY1]](s64), [[COPY2]](i32), implicit-def $scc
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s64) = COPY [[S_BFE_U64_]](s64)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[COPY3]](s64)
%0:_(s64) = COPY $sgpr0_sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir
index 59d7dcea9f2d9..b4bbe6fb88cd9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umax.mir
@@ -124,10 +124,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[UMAX:%[0-9]+]]:sgpr(s32) = G_UMAX [[ZEXT]], [[ZEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMAX]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[UMAX:%[0-9]+]]:sgpr(i32) = G_UMAX [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMAX]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -155,10 +155,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[UMAX:%[0-9]+]]:sgpr(s32) = G_UMAX [[ZEXT]], [[ZEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMAX]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[UMAX:%[0-9]+]]:sgpr(i32) = G_UMAX [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMAX]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -184,17 +184,17 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[BITCAST]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[BITCAST1]], [[C]]
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; CHECK-NEXT: [[UMAX:%[0-9]+]]:sgpr(s32) = G_UMAX [[AND]], [[AND1]]
- ; CHECK-NEXT: [[UMAX1:%[0-9]+]]:sgpr(s32) = G_UMAX [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[UMAX]](s32), [[UMAX1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[BITCAST]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[BITCAST1]], [[C]]
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[UMAX:%[0-9]+]]:sgpr(i32) = G_UMAX [[AND]], [[AND1]]
+ ; CHECK-NEXT: [[UMAX1:%[0-9]+]]:sgpr(i32) = G_UMAX [[LSHR]], [[LSHR1]]
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[UMAX]](i32), [[UMAX1]](i32)
; CHECK-NEXT: $sgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir
index fdb05f60e48e0..cce839a25d5ff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umin.mir
@@ -128,10 +128,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[UMIN:%[0-9]+]]:sgpr(s32) = G_UMIN [[ZEXT]], [[ZEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMIN]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[UMIN:%[0-9]+]]:sgpr(i32) = G_UMIN [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMIN]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -159,10 +159,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC1]](s16)
- ; CHECK-NEXT: [[UMIN:%[0-9]+]]:sgpr(s32) = G_UMIN [[ZEXT]], [[ZEXT1]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMIN]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC1]](s16)
+ ; CHECK-NEXT: [[UMIN:%[0-9]+]]:sgpr(i32) = G_UMIN [[ZEXT]], [[ZEXT1]]
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[UMIN]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(s32) = G_ANYEXT [[TRUNC2]](s16)
; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $sgpr0
@@ -188,17 +188,17 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[BITCAST]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[BITCAST1]], [[C]]
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; CHECK-NEXT: [[UMIN:%[0-9]+]]:sgpr(s32) = G_UMIN [[AND]], [[AND1]]
- ; CHECK-NEXT: [[UMIN1:%[0-9]+]]:sgpr(s32) = G_UMIN [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[UMIN]](s32), [[UMIN1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[BITCAST]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(i32) = G_AND [[BITCAST1]], [[C]]
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[UMIN:%[0-9]+]]:sgpr(i32) = G_UMIN [[AND]], [[AND1]]
+ ; CHECK-NEXT: [[UMIN1:%[0-9]+]]:sgpr(i32) = G_UMIN [[LSHR]], [[LSHR1]]
+ ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[UMIN]](i32), [[UMIN1]](i32)
; CHECK-NEXT: $sgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
%0:_(<2 x s16>) = COPY $sgpr0
%1:_(<2 x s16>) = COPY $sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir
index 3fa90e315fc27..2330a4a0fa3af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uniform-load-noclobber.mir
@@ -18,14 +18,14 @@ body: |
; GFX7-NEXT: %in_addr:sgpr(p1) = COPY $sgpr0_sgpr1
; GFX7-NEXT: %out_addr:sgpr(p1) = COPY $sgpr2_sgpr3
; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD %in_addr(p1) :: (load (<4 x s32>), align 4, addrspace 1)
- ; GFX7-NEXT: %cst16:sgpr(s64) = G_CONSTANT i64 16
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, %cst16(s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p1) :: (load (<4 x s32>) from unknown-address + 16, align 4, addrspace 1)
- ; GFX7-NEXT: %cst32:sgpr(s64) = G_CONSTANT i64 32
- ; GFX7-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, %cst32(s64)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
+ ; GFX7-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, [[C1]](i64)
; GFX7-NEXT: [[LOAD2:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD1]](p1) :: (load (<4 x s32>) from unknown-address + 32, align 4, addrspace 1)
- ; GFX7-NEXT: %cst48:sgpr(s64) = G_CONSTANT i64 48
- ; GFX7-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, %cst48(s64)
+ ; GFX7-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
+ ; GFX7-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, [[C2]](i64)
; GFX7-NEXT: [[LOAD3:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD2]](p1) :: (load (<4 x s32>) from unknown-address + 48, align 4, addrspace 1)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32), [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32), [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32), [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32), [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x s32>)
@@ -49,12 +49,15 @@ body: |
; GFX7-NEXT: %load0_3:sgpr(<4 x s32>), %load4_7:sgpr(<4 x s32>), %load8_11:sgpr(<4 x s32>), %load12_15:sgpr(<4 x s32>) = G_UNMERGE_VALUES %load(<16 x s32>)
; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load0_3(<4 x s32>)
; GFX7-NEXT: G_STORE [[COPY]](<4 x s32>), %out_addr(p1) :: (store (<4 x s32>), align 4, addrspace 1)
+ ; GFX7-NEXT: %cst16:sgpr(s64) = G_CONSTANT i64 16
; GFX7-NEXT: %out_addr_plus_16:sgpr(p1) = G_PTR_ADD %out_addr, %cst16(s64)
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load4_7(<4 x s32>)
; GFX7-NEXT: G_STORE [[COPY1]](<4 x s32>), %out_addr_plus_16(p1) :: (store (<4 x s32>), align 4, addrspace 1)
+ ; GFX7-NEXT: %cst32:sgpr(s64) = G_CONSTANT i64 32
; GFX7-NEXT: %out_addr_plus_32:sgpr(p1) = G_PTR_ADD %out_addr, %cst32(s64)
; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load8_11(<4 x s32>)
; GFX7-NEXT: G_STORE [[COPY2]](<4 x s32>), %out_addr_plus_32(p1) :: (store (<4 x s32>), align 4, addrspace 1)
+ ; GFX7-NEXT: %cst48:sgpr(s64) = G_CONSTANT i64 48
; GFX7-NEXT: %out_addr_plus_48:sgpr(p1) = G_PTR_ADD %out_addr, %cst48(s64)
; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load12_15(<4 x s32>)
; GFX7-NEXT: G_STORE [[COPY3]](<4 x s32>), %out_addr_plus_48(p1) :: (store (<4 x s32>), align 4, addrspace 1)
@@ -66,14 +69,14 @@ body: |
; GFX1010-NEXT: %in_addr:sgpr(p1) = COPY $sgpr0_sgpr1
; GFX1010-NEXT: %out_addr:sgpr(p1) = COPY $sgpr2_sgpr3
; GFX1010-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD %in_addr(p1) :: (load (<4 x s32>), align 4, addrspace 1)
- ; GFX1010-NEXT: %cst16:sgpr(s64) = G_CONSTANT i64 16
- ; GFX1010-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, %cst16(s64)
+ ; GFX1010-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GFX1010-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, [[C]](i64)
; GFX1010-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p1) :: (load (<4 x s32>) from unknown-address + 16, align 4, addrspace 1)
- ; GFX1010-NEXT: %cst32:sgpr(s64) = G_CONSTANT i64 32
- ; GFX1010-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, %cst32(s64)
+ ; GFX1010-NEXT: [[C1:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
+ ; GFX1010-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, [[C1]](i64)
; GFX1010-NEXT: [[LOAD2:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD1]](p1) :: (load (<4 x s32>) from unknown-address + 32, align 4, addrspace 1)
- ; GFX1010-NEXT: %cst48:sgpr(s64) = G_CONSTANT i64 48
- ; GFX1010-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, %cst48(s64)
+ ; GFX1010-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
+ ; GFX1010-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD %in_addr, [[C2]](i64)
; GFX1010-NEXT: [[LOAD3:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD2]](p1) :: (load (<4 x s32>) from unknown-address + 48, align 4, addrspace 1)
; GFX1010-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<16 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
; GFX1010-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32), [[UV8:%[0-9]+]]:vgpr(s32), [[UV9:%[0-9]+]]:vgpr(s32), [[UV10:%[0-9]+]]:vgpr(s32), [[UV11:%[0-9]+]]:vgpr(s32), [[UV12:%[0-9]+]]:vgpr(s32), [[UV13:%[0-9]+]]:vgpr(s32), [[UV14:%[0-9]+]]:vgpr(s32), [[UV15:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<16 x s32>)
@@ -97,12 +100,15 @@ body: |
; GFX1010-NEXT: %load0_3:sgpr(<4 x s32>), %load4_7:sgpr(<4 x s32>), %load8_11:sgpr(<4 x s32>), %load12_15:sgpr(<4 x s32>) = G_UNMERGE_VALUES %load(<16 x s32>)
; GFX1010-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load0_3(<4 x s32>)
; GFX1010-NEXT: G_STORE [[COPY]](<4 x s32>), %out_addr(p1) :: (store (<4 x s32>), align 4, addrspace 1)
+ ; GFX1010-NEXT: %cst16:sgpr(s64) = G_CONSTANT i64 16
; GFX1010-NEXT: %out_addr_plus_16:sgpr(p1) = G_PTR_ADD %out_addr, %cst16(s64)
; GFX1010-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load4_7(<4 x s32>)
; GFX1010-NEXT: G_STORE [[COPY1]](<4 x s32>), %out_addr_plus_16(p1) :: (store (<4 x s32>), align 4, addrspace 1)
+ ; GFX1010-NEXT: %cst32:sgpr(s64) = G_CONSTANT i64 32
; GFX1010-NEXT: %out_addr_plus_32:sgpr(p1) = G_PTR_ADD %out_addr, %cst32(s64)
; GFX1010-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load8_11(<4 x s32>)
; GFX1010-NEXT: G_STORE [[COPY2]](<4 x s32>), %out_addr_plus_32(p1) :: (store (<4 x s32>), align 4, addrspace 1)
+ ; GFX1010-NEXT: %cst48:sgpr(s64) = G_CONSTANT i64 48
; GFX1010-NEXT: %out_addr_plus_48:sgpr(p1) = G_PTR_ADD %out_addr, %cst48(s64)
; GFX1010-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load12_15(<4 x s32>)
; GFX1010-NEXT: G_STORE [[COPY3]](<4 x s32>), %out_addr_plus_48(p1) :: (store (<4 x s32>), align 4, addrspace 1)
@@ -138,8 +144,8 @@ body: |
; GFX7-NEXT: %ptr:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX7-NEXT: %out:sgpr(p1) = COPY $sgpr2_sgpr3
; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD %ptr(p4) :: (load (<4 x s32>), align 1, addrspace 4)
- ; GFX7-NEXT: %cst_16:sgpr(s64) = G_CONSTANT i64 16
- ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD %ptr, %cst_16(s64)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD %ptr, [[C]](i64)
; GFX7-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p4) :: (load (<4 x s32>) from unknown-address + 16, align 1, addrspace 4)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x s32>)
@@ -155,6 +161,7 @@ body: |
; GFX7-NEXT: %load0_3:sgpr(<4 x s32>), %load4_7:sgpr(<4 x s32>) = G_UNMERGE_VALUES %load(<8 x s32>)
; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load0_3(<4 x s32>)
; GFX7-NEXT: G_STORE [[COPY]](<4 x s32>), %out(p1) :: (store (<4 x s32>), align 32, addrspace 1)
+ ; GFX7-NEXT: %cst_16:sgpr(s64) = G_CONSTANT i64 16
; GFX7-NEXT: %out_plus_16:sgpr(p1) = G_PTR_ADD %out, %cst_16(s64)
; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load4_7(<4 x s32>)
; GFX7-NEXT: G_STORE [[COPY1]](<4 x s32>), %out_plus_16(p1) :: (store (<4 x s32>), align 32, addrspace 1)
@@ -166,8 +173,8 @@ body: |
; GFX1010-NEXT: %ptr:sgpr(p4) = COPY $sgpr0_sgpr1
; GFX1010-NEXT: %out:sgpr(p1) = COPY $sgpr2_sgpr3
; GFX1010-NEXT: [[LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD %ptr(p4) :: (load (<4 x s32>), align 1, addrspace 4)
- ; GFX1010-NEXT: %cst_16:sgpr(s64) = G_CONSTANT i64 16
- ; GFX1010-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD %ptr, %cst_16(s64)
+ ; GFX1010-NEXT: [[C:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
+ ; GFX1010-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = nuw inbounds G_PTR_ADD %ptr, [[C]](i64)
; GFX1010-NEXT: [[LOAD1:%[0-9]+]]:vgpr(<4 x s32>) = G_LOAD [[PTR_ADD]](p4) :: (load (<4 x s32>) from unknown-address + 16, align 1, addrspace 4)
; GFX1010-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
; GFX1010-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32), [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<8 x s32>)
@@ -183,6 +190,7 @@ body: |
; GFX1010-NEXT: %load0_3:sgpr(<4 x s32>), %load4_7:sgpr(<4 x s32>) = G_UNMERGE_VALUES %load(<8 x s32>)
; GFX1010-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load0_3(<4 x s32>)
; GFX1010-NEXT: G_STORE [[COPY]](<4 x s32>), %out(p1) :: (store (<4 x s32>), align 32, addrspace 1)
+ ; GFX1010-NEXT: %cst_16:sgpr(s64) = G_CONSTANT i64 16
; GFX1010-NEXT: %out_plus_16:sgpr(p1) = G_PTR_ADD %out, %cst_16(s64)
; GFX1010-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s32>) = COPY %load4_7(<4 x s32>)
; GFX1010-NEXT: G_STORE [[COPY1]](<4 x s32>), %out_plus_16(p1) :: (store (<4 x s32>), align 32, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
index c0be3c729e951..a6f2fb90aacf0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
@@ -16,10 +16,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C1]]
- ; CHECK-NEXT: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:sgpr(s32) = G_USUBE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C1]]
+ ; CHECK-NEXT: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:sgpr(i32) = G_USUBE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
@@ -42,9 +42,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY]], [[COPY3]], [[AMDGPU_COPY_VCC_SCC]]
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $sgpr0
@@ -115,9 +115,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:sgpr(s32) = G_USUBE [[COPY]], [[COPY1]], [[AND]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:sgpr(i32) = G_USUBE [[COPY]], [[COPY1]], [[AND]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = COPY $sgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
index 6732c8e964c87..b172f530efa75 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
@@ -13,7 +13,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[USUBO:%[0-9]+]]:sgpr(s32), [[USUBO1:%[0-9]+]]:sgpr(s32) = G_USUBO [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[USUBO:%[0-9]+]]:sgpr(s32), [[USUBO1:%[0-9]+]]:sgpr(i32) = G_USUBO [[COPY]], [[COPY1]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32), %3:_(s1) = G_USUBO %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-call.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-call.mir
index 5207d992ea74d..eca8dd5d1b221 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-call.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-waterfall-call.mir
@@ -23,8 +23,8 @@ body: |
; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(p0) = G_MERGE_VALUES [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[MV]](p0), %func_ptr
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .2:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
@@ -76,8 +76,8 @@ body: |
; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(p4) = G_MERGE_VALUES [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[MV]](p4), %func_ptr
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .2:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
@@ -129,8 +129,8 @@ body: |
; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(p0) = G_MERGE_VALUES [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[MV]](p0), %func_ptr
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .2:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
@@ -182,8 +182,8 @@ body: |
; CHECK-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[UV1]](s32)
; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(p4) = G_MERGE_VALUES [[INTRINSIC_CONVERGENT]](s32), [[INTRINSIC_CONVERGENT1]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[MV]](p4), %func_ptr
- ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(s64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
- ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](s64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[ICMP]](s1)
+ ; CHECK-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT2]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: .2:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir
index 892668ea7ab4a..e5cdf449bc4a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-widen-scalar-loads.mir
@@ -107,7 +107,7 @@ body: |
; GFX8: liveins: $sgpr0_sgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 8
; GFX8-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
;
@@ -115,7 +115,7 @@ body: |
; GFX9: liveins: $sgpr0_sgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX9-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 8
; GFX9-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
;
@@ -123,7 +123,7 @@ body: |
; GFX10: liveins: $sgpr0_sgpr1
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX10-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 8
; GFX10-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
%0:_(p1) = COPY $sgpr0_sgpr1
@@ -141,7 +141,7 @@ body: |
; GFX8: liveins: $sgpr0_sgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 16
; GFX8-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
;
@@ -149,7 +149,7 @@ body: |
; GFX9: liveins: $sgpr0_sgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX9-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 16
; GFX9-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
;
@@ -157,7 +157,7 @@ body: |
; GFX10: liveins: $sgpr0_sgpr1
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX10-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 16
; GFX10-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
%0:_(p1) = COPY $sgpr0_sgpr1
@@ -176,8 +176,8 @@ body: |
; GFX8: liveins: $sgpr0_sgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX8-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 255
+ ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 255
; GFX8-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX8-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
;
@@ -185,8 +185,8 @@ body: |
; GFX9: liveins: $sgpr0_sgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX9-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 255
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 255
; GFX9-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX9-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
;
@@ -194,8 +194,8 @@ body: |
; GFX10: liveins: $sgpr0_sgpr1
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX10-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 255
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 255
; GFX10-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX10-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
%0:_(p1) = COPY $sgpr0_sgpr1
@@ -213,8 +213,8 @@ body: |
; GFX8: liveins: $sgpr0_sgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX8-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX8-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
;
@@ -222,8 +222,8 @@ body: |
; GFX9: liveins: $sgpr0_sgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX9-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
; GFX9-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX9-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
;
@@ -231,8 +231,8 @@ body: |
; GFX10: liveins: $sgpr0_sgpr1
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX10-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
; GFX10-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX10-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
%0:_(p1) = COPY $sgpr0_sgpr1
@@ -312,7 +312,7 @@ body: |
; GFX8: liveins: $sgpr0_sgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 8
; GFX8-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
;
@@ -320,7 +320,7 @@ body: |
; GFX9: liveins: $sgpr0_sgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX9-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 8
; GFX9-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
;
@@ -328,7 +328,7 @@ body: |
; GFX10: liveins: $sgpr0_sgpr1
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
; GFX10-NEXT: [[SEXT_INREG:%[0-9]+]]:sgpr(s32) = G_SEXT_INREG [[LOAD]], 8
; GFX10-NEXT: S_ENDPGM 0, implicit [[SEXT_INREG]](s32)
%0:_(p1) = COPY $sgpr0_sgpr1
@@ -346,8 +346,8 @@ body: |
; GFX8: liveins: $sgpr0_sgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX8-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX8-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
;
@@ -355,8 +355,8 @@ body: |
; GFX9: liveins: $sgpr0_sgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX9-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
; GFX9-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX9-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
;
@@ -364,8 +364,8 @@ body: |
; GFX10: liveins: $sgpr0_sgpr1
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(s32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
- ; GFX10-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:sgpr(i32) = G_LOAD [[COPY]](p1) :: (invariant load (s32) from constant-pool, addrspace 4)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 65535
; GFX10-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[LOAD]], [[C]]
; GFX10-NEXT: S_ENDPGM 0, implicit [[AND]](s32)
%0:_(p1) = COPY $sgpr0_sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
index 0ebb1344d407d..e0d17574a62be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
@@ -88,10 +88,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY1]](s32), [[C]]
- ; CHECK-NEXT: [[XOR:%[0-9]+]]:sgpr(s32) = G_XOR [[ICMP]], [[ICMP1]]
- ; CHECK-NEXT: S_NOP 0, implicit [[XOR]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY1]](s32), [[C]]
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:sgpr(i32) = G_XOR [[ICMP]], [[ICMP1]]
+ ; CHECK-NEXT: S_NOP 0, implicit [[XOR]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s32) = G_CONSTANT i32 0
@@ -143,10 +143,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](i32)
; CHECK-NEXT: [[XOR:%[0-9]+]]:vcc(s1) = G_XOR [[AMDGPU_COPY_VCC_SCC]], [[ICMP1]]
; CHECK-NEXT: S_NOP 0, implicit [[XOR]](s1)
%0:_(s32) = COPY $sgpr0
@@ -169,8 +169,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[XOR:%[0-9]+]]:sgpr(s32) = G_XOR [[COPY]], [[COPY1]]
- ; CHECK-NEXT: S_NOP 0, implicit [[XOR]](s32)
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:sgpr(i32) = G_XOR [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: S_NOP 0, implicit [[XOR]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_TRUNC %0
@@ -192,9 +192,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[XOR:%[0-9]+]]:sgpr(s32) = G_XOR [[COPY]], [[ICMP]]
- ; CHECK-NEXT: S_NOP 0, implicit [[XOR]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:sgpr(i32) = G_XOR [[COPY]], [[ICMP]]
+ ; CHECK-NEXT: S_NOP 0, implicit [[XOR]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_TRUNC %0
@@ -828,8 +828,8 @@ body: |
; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C1]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[C1]](i32)
; CHECK-NEXT: [[XOR:%[0-9]+]]:vcc(s1) = G_XOR [[ICMP]], [[AMDGPU_COPY_VCC_SCC]]
; CHECK-NEXT: S_NOP 0, implicit [[XOR]](s1)
%0:_(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir
index d4baa5fb864fc..81104df036869 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-zext.mir
@@ -12,8 +12,8 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[C]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s64) = G_ZEXT %0
...
@@ -30,9 +30,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[ZEXT]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:sgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[ZEXT]](i32), [[C]](i32)
%0:_(s32) = COPY $sgpr0
%1:_(s16) = G_TRUNC %0
%2:_(s64) = G_ZEXT %1
@@ -49,8 +49,8 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[C]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s64) = G_ZEXT %0
...
@@ -67,12 +67,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C]], [[C1]]
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](s32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(i32) = G_SELECT [[AND]](i32), [[C]], [[C1]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](i32)
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s16) = G_CONSTANT i16 255
; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s16) = G_AND [[TRUNC]], [[C2]]
%0:_(s32) = COPY $sgpr0
@@ -95,11 +95,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C]], [[C1]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -118,12 +119,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
- ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(i32) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[ICMP]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 1
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s32) = COPY $sgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -187,10 +188,10 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](i32), [[C1]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s1) = G_ICMP intpred(eq), %0, %1
@@ -208,11 +209,11 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C]], [[C1]]
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(i32) = G_SELECT [[AND]](i32), [[C]], [[C1]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[SELECT]](i32)
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s16) = G_CONSTANT i16 255
; CHECK-NEXT: [[AND1:%[0-9]+]]:sgpr(s16) = G_AND [[TRUNC]], [[C2]]
%0:_(s32) = COPY $sgpr0
@@ -233,10 +234,11 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](s32), [[C]], [[C1]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s1) = G_TRUNC %0
%2:_(s32) = G_ZEXT %1
@@ -253,11 +255,11 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(i32) = G_AND [[COPY]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 1
; CHECK-NEXT: [[C2:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](s32), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[AND]](i32), [[C1]], [[C2]]
%0:_(s32) = COPY $sgpr0
%1:_(s1) = G_TRUNC %0
%2:_(s64) = G_ZEXT %1
@@ -322,8 +324,10 @@ body: |
; CHECK-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[C]], [[C1]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[C1]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(i32) = G_SELECT [[ICMP]](s1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](i32), [[C3]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s1) = G_TRUNC %0
%2:_(s64) = G_ZEXT %1
@@ -341,9 +345,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:vgpr(s32) = G_ZEXT [[TRUNC]](s16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[ZEXT]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:vgpr(i32) = G_ZEXT [[TRUNC]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:vgpr(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[ZEXT]](i32), [[C]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s16) = G_TRUNC %0
%2:_(s64) = G_ZEXT %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 03cf8c1ccaa23..2fb29190421a7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -93,17 +93,22 @@ define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
;
; GFX8-LABEL: s_saddsat_i7:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b32 s1, s1, 9
; GFX8-NEXT: s_lshl_b32 s0, s0, 9
+; GFX8-NEXT: s_sext_i32_i16 s2, s0
+; GFX8-NEXT: s_max_i32 s3, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 9
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 9
-; GFX8-NEXT: s_lshr_b32 s0, s0, 9
+; GFX8-NEXT: s_sub_i32 s3, 0x7fff, s3
+; GFX8-NEXT: s_max_i32 s1, s2, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s2, s3
+; GFX8-NEXT: s_min_i32 s1, s1, s2
; GFX8-NEXT: s_add_i32 s0, s0, s1
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_not_b32_e32 v0, 63
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, 63
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s0, s0, 9
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_i7:
@@ -112,39 +117,39 @@ define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
; GFX9-NEXT: s_lshl_b32 s0, s0, 9
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_add_i16 v0, s0, v0 clamp
-; GFX9-NEXT: v_mov_b32_e32 v1, 9
-; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 9
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_saddsat_i7:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshl_b32 s1, s1, 9
; GFX10-NEXT: s_lshl_b32 s0, s0, 9
-; GFX10-NEXT: v_mov_b32_e32 v1, 9
+; GFX10-NEXT: s_lshl_b32 s1, s1, 9
; GFX10-NEXT: v_add_nc_i16 v0, s0, s1 clamp
-; GFX10-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_sext_i32_i16 s0, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 9
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_saddsat_i7:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-TRUE16-NEXT: v_add_nc_i16 v0.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v0, 9, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 9
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_saddsat_i7:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-FAKE16-NEXT: v_add_nc_i16 v0, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v0, 9, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 9
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i7 @llvm.sadd.sat.i7(i7 %lhs, i7 %rhs)
ret i7 %result
@@ -237,14 +242,22 @@ define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
;
; GFX8-LABEL: s_saddsat_i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x80000
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s2, s0
+; GFX8-NEXT: s_max_i32 s3, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sub_i32 s3, 0x7fff, s3
+; GFX8-NEXT: s_max_i32 s1, s2, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s2, s3
+; GFX8-NEXT: s_min_i32 s1, s1, s2
; GFX8-NEXT: s_add_i32 s0, s0, s1
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7f
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s0, s0, 8
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_i8:
@@ -253,39 +266,39 @@ define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshl_b32 s0, s0, 8
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_add_i16 v0, s0, v0 clamp
-; GFX9-NEXT: v_mov_b32_e32 v1, 8
-; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 8
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_saddsat_i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_lshl_b32 s0, s0, 8
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: v_add_nc_i16 v0, s0, s1 clamp
-; GFX10-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_sext_i32_i16 s0, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 8
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_saddsat_i8:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-TRUE16-NEXT: v_add_nc_i16 v0.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_saddsat_i8:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-FAKE16-NEXT: v_add_nc_i16 v0, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v0, 8, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 8
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i8 @llvm.sadd.sat.i8(i8 %lhs, i8 %rhs)
ret i8 %result
@@ -450,23 +463,44 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX8-LABEL: s_saddsat_v2i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_sext_i32_i16 s2, s1
-; GFX8-NEXT: s_sext_i32_i16 s3, s0
-; GFX8-NEXT: s_ashr_i32 s2, s2, 8
-; GFX8-NEXT: s_ashr_i32 s3, s3, 8
-; GFX8-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x80000
-; GFX8-NEXT: s_add_i32 s3, s3, s2
-; GFX8-NEXT: s_sext_i32_i16 s2, s3
-; GFX8-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s4, s0
+; GFX8-NEXT: s_max_i32 s5, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_lshr_b32 s3, s1, 8
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT: s_max_i32 s1, s4, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s4, s5
+; GFX8-NEXT: s_min_i32 s1, s1, s4
; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: v_med3_i32 v2, s2, v0, v1
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: s_sext_i32_i16 s3, s1
+; GFX8-NEXT: s_max_i32 s4, s3, 0
+; GFX8-NEXT: s_min_i32 s3, s3, 0
+; GFX8-NEXT: s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sub_i32 s4, 0x7fff, s4
+; GFX8-NEXT: s_max_i32 s2, s3, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s3, s4
+; GFX8-NEXT: s_min_i32 s2, s2, s3
+; GFX8-NEXT: s_add_i32 s1, s1, s2
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s1, s1, 8
+; GFX8-NEXT: s_ashr_i32 s0, s0, 8
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_v2i8:
@@ -498,14 +532,31 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX11-TRUE16-LABEL: s_saddsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s1, 8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s0, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v0.l, s3, s2 clamp
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v1.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0105
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: v_pk_add_i16 v0, s0, s1 clamp
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s1, 0x80008
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s2, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s2, s1
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_saddsat_v2i8:
@@ -2501,19 +2552,18 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
;
; GFX8-LABEL: s_saddsat_i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_add_i32 s4, s0, s1
; GFX8-NEXT: s_sext_i32_i16 s2, s0
-; GFX8-NEXT: s_sext_i32_i16 s0, s1
-; GFX8-NEXT: s_sext_i32_i16 s5, s4
-; GFX8-NEXT: s_cmp_lt_i32 s0, 0
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s5, s2
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: s_ashr_i32 s2, s5, 15
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX8-NEXT: s_cselect_b32 s0, s2, s4
+; GFX8-NEXT: s_max_i32 s3, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sub_i32 s3, 0x7fff, s3
+; GFX8-NEXT: s_max_i32 s1, s2, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s2, s3
+; GFX8-NEXT: s_min_i32 s1, s1, s2
+; GFX8-NEXT: s_add_i32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_i16:
@@ -3754,110 +3804,87 @@ define i48 @v_saddsat_i48(i48 %lhs, i48 %rhs) {
; GFX6-LABEL: v_saddsat_i48:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 16
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 16
-; GFX6-NEXT: v_ashr_i64 v[2:3], v[2:3], 16
-; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_mov_b32 s4, -1
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT: s_movk_i32 s5, 0x7fff
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: s_mov_b32 s4, 0
-; GFX6-NEXT: s_movk_i32 s5, 0x8000
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v1, v3, vcc
+; GFX6-NEXT: v_bfe_i32 v5, v6, 0, 16
+; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[0:1]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xffff8000, v0
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 15, v5
+; GFX6-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_i48:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
-; GFX8-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_mov_b32 s4, -1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8-NEXT: s_movk_i32 s5, 0x7fff
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX8-NEXT: s_mov_b32 s4, 0
-; GFX8-NEXT: s_movk_i32 s5, 0x8000
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, v1, v3, vcc
+; GFX8-NEXT: v_bfe_i32 v5, v6, 0, 16
+; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[0:1]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0xffff8000, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 15, v5
+; GFX8-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_i48:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
+; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
+; GFX9-NEXT: s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: s_mov_b32 s4, -1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9-NEXT: s_movk_i32 s5, 0x7fff
-; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX9-NEXT: s_mov_b32 s4, 0
-; GFX9-NEXT: s_movk_i32 s5, 0x8000
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_saddsat_i48:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: s_mov_b32 s4, -1
-; GFX10-NEXT: s_movk_i32 s5, 0x7fff
-; GFX10-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX10-NEXT: v_cmp_lt_i64_e64 s4, v[4:5], v[0:1]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
+; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_movk_i32 s5, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fff, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff8000, v1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_saddsat_i48:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, -1
-; GFX11-NEXT: s_movk_i32 s1, 0x7fff
-; GFX11-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
+; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX11-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX11-NEXT: v_cmp_lt_i64_e64 s0, v[4:5], v[0:1]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
+; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v6 :: v_dual_cndmask_b32 v1, v5, v1
; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_movk_i32 s1, 0x8000
-; GFX11-NEXT: v_dual_cndmask_b32 v1, 0x7fff, v1 :: v_dual_cndmask_b32 v0, -1, v0
-; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_dual_cndmask_b32 v0, 0, v0 :: v_dual_cndmask_b32 v1, 0xffff8000, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i48 @llvm.sadd.sat.i48(i48 %lhs, i48 %rhs)
ret i48 %result
@@ -3958,49 +3985,39 @@ define amdgpu_ps i48 @s_saddsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
define amdgpu_ps <2 x float> @saddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX6-LABEL: saddsat_i48_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc
-; GFX6-NEXT: s_mov_b32 s0, -1
-; GFX6-NEXT: s_movk_i32 s1, 0x7fff
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: s_mov_b32 s0, 0
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v3, v1, vcc
+; GFX6-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX6-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[2:3]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[0:1]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xffff8000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i48_sv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc
-; GFX8-NEXT: s_mov_b32 s0, -1
-; GFX8-NEXT: s_movk_i32 s1, 0x7fff
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX8-NEXT: s_mov_b32 s0, 0
-; GFX8-NEXT: s_movk_i32 s1, 0x8000
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, v3, v1, vcc
+; GFX8-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX8-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[0:1]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xffff8000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: ; return to shader part epilog
;
@@ -4008,23 +4025,17 @@ define amdgpu_ps <2 x float> @saddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[0:1], 0, v[0:1]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
+; GFX9-NEXT: s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
-; GFX9-NEXT: s_mov_b32 s0, -1
-; GFX9-NEXT: s_movk_i32 s1, 0x7fff
-; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX9-NEXT: s_mov_b32 s0, 0
-; GFX9-NEXT: s_movk_i32 s1, 0x8000
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -4032,20 +4043,16 @@ define amdgpu_ps <2 x float> @saddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX10-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, s0, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[2:3]
+; GFX10-NEXT: v_cmp_gt_i64_e64 s0, 0, v[0:1]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX10-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s0, -1
-; GFX10-NEXT: s_movk_i32 s1, 0x7fff
-; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_movk_i32 s1, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fff, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff8000, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: ; return to shader part epilog
;
@@ -4053,18 +4060,15 @@ define amdgpu_ps <2 x float> @saddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX11: ; %bb.0:
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX11-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
+; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, s0, v0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[2:3]
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[0:1]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX11-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX11-NEXT: s_mov_b32 s0, -1
-; GFX11-NEXT: s_movk_i32 s1, 0x7fff
-; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_movk_i32 s1, 0x8000
-; GFX11-NEXT: v_dual_cndmask_b32 v1, 0x7fff, v1 :: v_dual_cndmask_b32 v0, -1, v0
-; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_dual_cndmask_b32 v0, 0, v0 :: v_dual_cndmask_b32 v1, 0xffff8000, v1
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i48 @llvm.sadd.sat.i48(i48 %lhs, i48 %rhs)
@@ -4076,49 +4080,43 @@ define amdgpu_ps <2 x float> @saddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
define amdgpu_ps <2 x float> @saddsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX6-LABEL: saddsat_i48_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
-; GFX6-NEXT: s_mov_b32 s0, -1
-; GFX6-NEXT: s_movk_i32 s1, 0x7fff
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: s_mov_b32 s0, 0
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v1, v3, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
+; GFX6-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[0:1], v[2:3], v[0:1]
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: s_cselect_b64 s[2:3], exec, 0
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xffff8000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i48_vs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
-; GFX8-NEXT: s_mov_b32 s0, -1
-; GFX8-NEXT: s_movk_i32 s1, 0x7fff
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX8-NEXT: s_mov_b32 s0, 0
-; GFX8-NEXT: s_movk_i32 s1, 0x8000
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, v1, v3, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
+; GFX8-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: s_cselect_b64 s[2:3], exec, 0
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xffff8000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: ; return to shader part epilog
;
@@ -4126,23 +4124,19 @@ define amdgpu_ps <2 x float> @saddsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT: v_cmp_lt_i64_e64 s[0:1], s[0:1], 0
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], exec, 0
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
+; GFX9-NEXT: s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
-; GFX9-NEXT: s_mov_b32 s0, -1
-; GFX9-NEXT: s_movk_i32 s1, 0x7fff
-; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX9-NEXT: s_mov_b32 s0, 0
-; GFX9-NEXT: s_movk_i32 s1, 0x8000
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -4150,20 +4144,18 @@ define amdgpu_ps <2 x float> @saddsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX10-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e64 s0, s[0:1], 0
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s0, 0
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX10-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s0, -1
-; GFX10-NEXT: s_movk_i32 s1, 0x7fff
-; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_movk_i32 s1, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fff, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff8000, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: ; return to shader part epilog
;
@@ -4171,18 +4163,17 @@ define amdgpu_ps <2 x float> @saddsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX11: ; %bb.0:
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX11-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
+; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v0, s0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
+; GFX11-NEXT: v_cmp_lt_i64_e64 s0, s[0:1], 0
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX11-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX11-NEXT: s_mov_b32 s0, -1
-; GFX11-NEXT: s_movk_i32 s1, 0x7fff
-; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_movk_i32 s1, 0x8000
-; GFX11-NEXT: v_dual_cndmask_b32 v1, 0x7fff, v1 :: v_dual_cndmask_b32 v0, -1, v0
-; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_dual_cndmask_b32 v0, 0, v0 :: v_dual_cndmask_b32 v1, 0xffff8000, v1
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i48 @llvm.sadd.sat.i48(i48 %lhs, i48 %rhs)
@@ -4197,12 +4188,13 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v0, v2
; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v1, v3, vcc
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v3
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v5
-; GFX6-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v5, -v1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[0:1]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_i64:
@@ -4210,12 +4202,13 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v0, v2
; GFX8-NEXT: v_addc_u32_e32 v5, vcc, v1, v3, vcc
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v5
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v5, -v1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[0:1]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x80000000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_i64:
@@ -4224,11 +4217,12 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v3, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v3
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v5
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v5, -v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_saddsat_i64:
@@ -4236,12 +4230,13 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: v_cmp_gt_i32_e64 s4, 0, v3
+; GFX10-NEXT: v_cmp_gt_i64_e64 s4, 0, v[2:3]
+; GFX10-NEXT: v_ashrrev_i32_e32 v6, 31, v5
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v5
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
; GFX10-NEXT: s_xor_b32 vcc_lo, s4, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_saddsat_i64:
@@ -4249,12 +4244,12 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2
; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v3
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX11-NEXT: v_ashrrev_i32_e32 v6, 31, v5
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
; GFX11-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v6 :: v_dual_cndmask_b32 v1, v5, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
ret i64 %result
@@ -4344,12 +4339,13 @@ define amdgpu_ps <2 x float> @saddsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
; GFX6-NEXT: v_mov_b32_e32 v3, s1
; GFX6-NEXT: v_add_i32_e32 v2, vcc, s0, v0
; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v3
-; GFX6-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[2:3]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[0:1]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i64_sv:
@@ -4357,12 +4353,13 @@ define amdgpu_ps <2 x float> @saddsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v0
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v3
-; GFX8-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[0:1]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x80000000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i64_sv:
@@ -4371,35 +4368,37 @@ define amdgpu_ps <2 x float> @saddsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v0
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[0:1], 0, v[0:1]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: saddsat_i64_sv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, s0, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[2:3]
-; GFX10-NEXT: v_cmp_gt_i32_e64 s0, 0, v1
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX10-NEXT: v_cmp_gt_i64_e64 s0, 0, v[0:1]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX10-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: saddsat_i64_sv:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, s0, v0
; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[2:3]
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v1
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[0:1]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX11-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
%cast = bitcast i64 %result to <2 x float>
@@ -4411,68 +4410,77 @@ define amdgpu_ps <2 x float> @saddsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
; GFX6: ; %bb.0:
; GFX6-NEXT: v_mov_b32_e32 v3, s1
; GFX6-NEXT: v_add_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
-; GFX6-NEXT: s_cmp_lt_i32 s1, 0
-; GFX6-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v3
-; GFX6-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[0:1], v[2:3], v[0:1]
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: s_cselect_b64 s[2:3], exec, 0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i64_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
-; GFX8-NEXT: s_cmp_lt_i32 s1, 0
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v3
-; GFX8-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: s_cselect_b64 s[2:3], exec, 0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x80000000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[2:3], s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i64_vs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT: v_cmp_lt_i64_e64 s[0:1], s[0:1], 0
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
-; GFX9-NEXT: s_cmp_lt_i32 s1, 0
-; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], exec, 0
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: saddsat_i64_vs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, s0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: s_cmp_lt_i32 s1, 0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
+; GFX10-NEXT: v_cmp_lt_i64_e64 s0, s[0:1], 0
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[0:1]
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX10-NEXT: s_cmp_lg_u32 s0, 0
+; GFX10-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX10-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: saddsat_i64_vs:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v0, s0
; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
-; GFX11-NEXT: s_cmp_lt_i32 s1, 0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
+; GFX11-NEXT: v_cmp_lt_i64_e64 s0, s[0:1], 0
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[0:1]
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-NEXT: s_cselect_b32 s0, exec_lo, 0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX11-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
%cast = bitcast i64 %result to <2 x float>
@@ -4485,20 +4493,23 @@ define <2 x i64> @v_saddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v8, vcc, v0, v4
; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v1, v5, vcc
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v9
-; GFX6-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v9, -v1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v9
+; GFX6-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v0, v1
+; GFX6-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v2, v6
; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v3, v7, vcc
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v7
-; GFX6-NEXT: v_ashrrev_i32_e32 v3, 31, v5
-; GFX6-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v5, -v3, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[2:3]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[6:7]
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0x80000000, v2
+; GFX6-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_v2i64:
@@ -4506,20 +4517,23 @@ define <2 x i64> @v_saddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v8, vcc, v0, v4
; GFX8-NEXT: v_addc_u32_e32 v9, vcc, v1, v5, vcc
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v5
-; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v9
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v9, -v1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[8:9], v[0:1]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v9
+; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v2, v6
; GFX8-NEXT: v_addc_u32_e32 v5, vcc, v3, v7, vcc
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v7
-; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v5
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, -v3, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[2:3]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[6:7]
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x80000000, v2
+; GFX8-NEXT: s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_v2i64:
@@ -4528,19 +4542,21 @@ define <2 x i64> @v_saddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v5, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v5
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v9
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v9
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, -v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v2, v6
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v3, v7, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v7
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v5
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[6:7]
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX9-NEXT: v_add_u32_e32 v3, 0x80000000, v2
; GFX9-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, -v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_saddsat_v2i64:
@@ -4550,18 +4566,20 @@ define <2 x i64> @v_saddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v5, vcc_lo
; GFX10-NEXT: v_add_co_u32 v10, vcc_lo, v2, v6
; GFX10-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT: v_ashrrev_i32_e32 v12, 31, v9
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_gt_i32_e64 s4, 0, v5
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v9
+; GFX10-NEXT: v_cmp_gt_i64_e64 s4, 0, v[4:5]
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v11
; GFX10-NEXT: v_cmp_lt_i64_e64 s5, v[10:11], v[2:3]
-; GFX10-NEXT: v_cmp_gt_i32_e64 s6, 0, v7
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v11
+; GFX10-NEXT: v_cmp_gt_i64_e64 s6, 0, v[6:7]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v12
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v4
; GFX10-NEXT: s_xor_b32 vcc_lo, s4, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v12, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo
; GFX10-NEXT: s_xor_b32 vcc_lo, s6, s5
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, -v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v10, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_saddsat_v2i64:
@@ -4571,18 +4589,18 @@ define <2 x i64> @v_saddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX11-NEXT: v_add_co_ci_u32_e64 v9, null, v1, v5, vcc_lo
; GFX11-NEXT: v_add_co_u32 v10, vcc_lo, v2, v6
; GFX11-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v7, vcc_lo
+; GFX11-NEXT: v_ashrrev_i32_e32 v12, 31, v9
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[8:9], v[0:1]
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v5
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v9
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[4:5]
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v11
; GFX11-NEXT: v_cmp_lt_i64_e64 s1, v[10:11], v[2:3]
-; GFX11-NEXT: v_cmp_gt_i32_e64 s2, 0, v7
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v11
+; GFX11-NEXT: v_cmp_gt_i64_e64 s2, 0, v[6:7]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v12
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v4
; GFX11-NEXT: s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v8, v12 :: v_dual_cndmask_b32 v1, v9, v1
; GFX11-NEXT: s_xor_b32 vcc_lo, s2, s1
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, -v3, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v10, v4 :: v_dual_cndmask_b32 v3, v11, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
ret <2 x i64> %result
@@ -4870,18 +4888,27 @@ define amdgpu_ps <4 x float> @saddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v4, v1, vcc
; GFX6-NEXT: v_mov_b32_e32 v4, s2
; GFX6-NEXT: v_mov_b32_e32 v5, s3
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v4, v2, vcc
-; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v5, v3, vcc
-; GFX6-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX6-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX6-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX6-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v2, vcc
+; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v3, vcc
+; GFX6-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 31, v5
+; GFX6-NEXT: v_cndmask_b32_e64 v2, v7, 0, vcc
+; GFX6-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX6-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000000, v3
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i128_sv:
@@ -4891,18 +4918,26 @@ define amdgpu_ps <4 x float> @saddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v4, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: v_mov_b32_e32 v5, s3
-; GFX8-NEXT: v_addc_u32_e32 v2, vcc, v4, v2, vcc
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, v5, v3, vcc
-; GFX8-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX8-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, v4, v2, vcc
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, v5, v3, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v5
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, 0, vcc
+; GFX8-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000000, v3
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i128_sv:
@@ -4912,55 +4947,103 @@ define amdgpu_ps <4 x float> @saddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
; GFX9-NEXT: v_mov_b32_e32 v4, s2
; GFX9-NEXT: v_mov_b32_e32 v5, s3
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v2, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v5, v3, vcc
-; GFX9-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX9-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v2, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v3, vcc
+; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v5
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v7, 0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-NEXT: v_add_u32_e32 v6, 0x80000000, v3
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: saddsat_i128_sv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, s3, v3, vcc_lo
-; GFX10-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX10-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX10-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[2:3], v[4:5]
+; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[4:5]
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v5
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc_lo
+; GFX10-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-NEXT: v_add_nc_u32_e32 v6, 0x80000000, v3
+; GFX10-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: saddsat_i128_sv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e64 v4, null, s3, v3, vcc_lo
-; GFX11-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX11-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX11-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: saddsat_i128_sv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, s0, v0
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v7, null, s3, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[4:5]
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e64 s1, s[2:3], v[6:7]
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s2, s[2:3], v[6:7]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[2:3]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v2
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v4, v2 :: v_dual_cndmask_b32 v3, v7, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: saddsat_i128_sv:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v5, null, s3, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[2:3], v[4:5]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[4:5]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[2:3]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v5
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc_lo
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x80000000, v3
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v2, v4, v3 :: v_dual_cndmask_b32 v3, v5, v6
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i128 @llvm.sadd.sat.i128(i128 %lhs, i128 %rhs)
%cast = bitcast i128 %result to <4 x float>
ret <4 x float> %cast
@@ -4969,102 +5052,173 @@ define amdgpu_ps <4 x float> @saddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
define amdgpu_ps <4 x float> @saddsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
; GFX6-LABEL: saddsat_i128_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v4, s1
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
-; GFX6-NEXT: v_mov_b32_e32 v4, s2
-; GFX6-NEXT: v_mov_b32_e32 v5, s3
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
-; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v3, v5, vcc
-; GFX6-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX6-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX6-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX6-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX6-NEXT: v_mov_b32_e32 v5, s1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, s0, v0
+; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v1, v5, vcc
+; GFX6-NEXT: v_mov_b32_e32 v6, s2
+; GFX6-NEXT: v_mov_b32_e32 v7, s3
+; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v3, v7, vcc
+; GFX6-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT: s_cselect_b32 s4, 1, 0
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT: s_cselect_b32 s0, 0, s4
+; GFX6-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; GFX6-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0x80000000, v2
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i128_vs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v4, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s3
-; GFX8-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, v3, v5, vcc
-; GFX8-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX8-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, v1, v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v6, s2
+; GFX8-NEXT: v_mov_b32_e32 v7, s3
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, v3, v7, vcc
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8-NEXT: s_cmp_eq_u64 s[2:3], 0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT: s_cselect_b32 s0, 0, s0
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; GFX8-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x80000000, v2
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i128_vs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s3
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v3, v5, vcc
-; GFX9-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX9-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v6, s2
+; GFX9-NEXT: v_mov_b32_e32 v7, s3
+; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v2, v6, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v3, v7, vcc
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT: v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: s_cmp_eq_u64 s[2:3], 0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT: s_cselect_b32 s0, 0, s0
+; GFX9-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; GFX9-NEXT: v_add_u32_e32 v3, 0x80000000, v2
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: saddsat_i128_vs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, s3, v3, vcc_lo
-; GFX10-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX10-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX10-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX10-NEXT: v_cmp_lt_i64_e64 s0, s[2:3], 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[6:7], v[2:3]
+; GFX10-NEXT: s_cmp_lg_u32 s0, 0
+; GFX10-NEXT: s_cselect_b32 s0, 1, 0
+; GFX10-NEXT: s_cmp_eq_u64 s[2:3], 0
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[2:3]
+; GFX10-NEXT: s_cselect_b32 s0, 0, s0
+; GFX10-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v2
+; GFX10-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX10-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: saddsat_i128_vs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e64 v4, null, s3, v3, vcc_lo
-; GFX11-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX11-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX11-NEXT: v_xor_b32_e32 v3, s1, v3
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: saddsat_i128_vs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, v0, s0
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v7, null, s3, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmp_lt_i64_e64 s4, s[2:3], 0
+; GFX11-TRUE16-NEXT: v_cmp_lt_i64_e64 s0, v[6:7], v[2:3]
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s1, v[6:7], v[2:3]
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_eq_u64 s[2:3], 0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v2
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 0, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s1
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, s0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v4, v2 :: v_dual_cndmask_b32 v3, v7, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: saddsat_i128_vs:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, v0, s0
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v7, null, s3, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmp_lt_i64_e64 s0, s[2:3], 0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[6:7], v[2:3]
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_eq_u64 s[2:3], 0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[2:3]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 0, s0
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v1, v0 :: v_dual_add_nc_u32 v3, 0x80000000, v2
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v4, v2 :: v_dual_cndmask_b32 v3, v7, v3
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i128 @llvm.sadd.sat.i128(i128 %lhs, i128 %rhs)
%cast = bitcast i128 %result to <4 x float>
ret <4 x float> %cast
@@ -5074,157 +5228,298 @@ define <2 x i128> @v_saddsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
; GFX6-LABEL: v_saddsat_v2i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v8
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v9, vcc
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v2, v10, vcc
-; GFX6-NEXT: v_addc_u32_e32 v8, vcc, v3, v11, vcc
-; GFX6-NEXT: v_xor_b32_e32 v9, v3, v8
-; GFX6-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX6-NEXT: v_bfi_b32 v3, v3, 0, v9
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v8, -v9, vcc
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v12
-; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc
-; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v6, v14, vcc
-; GFX6-NEXT: v_addc_u32_e32 v8, vcc, v7, v15, vcc
-; GFX6-NEXT: v_xor_b32_e32 v9, v7, v8
-; GFX6-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX6-NEXT: v_bfi_b32 v7, v7, 0, v9
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v7
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v7, v8, -v9, vcc
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v0, v8
+; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v1, v9, vcc
+; GFX6-NEXT: v_addc_u32_e32 v16, vcc, v2, v10, vcc
+; GFX6-NEXT: v_addc_u32_e32 v17, vcc, v3, v11, vcc
+; GFX6-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[0:1]
+; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[16:17], v[2:3]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[16:17], v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v17
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[10:11]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX6-NEXT: v_xor_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v2, v1
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v9, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v4, v12
+; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v5, v13, vcc
+; GFX6-NEXT: v_addc_u32_e32 v10, vcc, v6, v14, vcc
+; GFX6-NEXT: v_addc_u32_e32 v11, vcc, v7, v15, vcc
+; GFX6-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX6-NEXT: v_ashrrev_i32_e32 v6, 31, v11
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[14:15]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc
+; GFX6-NEXT: v_xor_b32_e32 v4, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 1, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0x80000000, v6
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v8
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v9, vcc
-; GFX8-NEXT: v_addc_u32_e32 v2, vcc, v2, v10, vcc
-; GFX8-NEXT: v_addc_u32_e32 v8, vcc, v3, v11, vcc
-; GFX8-NEXT: v_xor_b32_e32 v9, v3, v8
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_bfi_b32 v3, v3, 0, v9
-; GFX8-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v8, -v9, vcc
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v12
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc
-; GFX8-NEXT: v_addc_u32_e32 v6, vcc, v6, v14, vcc
-; GFX8-NEXT: v_addc_u32_e32 v8, vcc, v7, v15, vcc
-; GFX8-NEXT: v_xor_b32_e32 v9, v7, v8
-; GFX8-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_bfi_b32 v7, v7, 0, v9
-; GFX8-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v7, v8, -v9, vcc
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v0, v8
+; GFX8-NEXT: v_addc_u32_e32 v9, vcc, v1, v9, vcc
+; GFX8-NEXT: v_addc_u32_e32 v16, vcc, v2, v10, vcc
+; GFX8-NEXT: v_addc_u32_e32 v17, vcc, v3, v11, vcc
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[16:17], v[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[16:17], v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v17
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[10:11]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX8-NEXT: v_xor_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v2, v1
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v4, v12
+; GFX8-NEXT: v_addc_u32_e32 v9, vcc, v5, v13, vcc
+; GFX8-NEXT: v_addc_u32_e32 v10, vcc, v6, v14, vcc
+; GFX8-NEXT: v_addc_u32_e32 v11, vcc, v7, v15, vcc
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX8-NEXT: v_ashrrev_i32_e32 v6, 31, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[14:15]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc
+; GFX8-NEXT: v_xor_b32_e32 v4, v5, v4
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x80000000, v6
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v8
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v9, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v10, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v3, v11, vcc
-; GFX9-NEXT: v_xor_b32_e32 v9, v3, v8
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_bfi_b32 v3, v3, 0, v9
-; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, -v9, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v12
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v13, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v14, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v7, v15, vcc
-; GFX9-NEXT: v_xor_b32_e32 v9, v7, v8
-; GFX9-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_bfi_b32 v7, v7, 0, v9
-; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v7
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v7, v8, -v9, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v0, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v9, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v16, vcc, v2, v10, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v17, vcc, v3, v11, vcc
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[16:17], v[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[16:17], v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT: v_add_u32_e32 v3, 0x80000000, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v4, v12
+; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v5, v13, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v6, v14, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v7, v15, vcc
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX9-NEXT: v_ashrrev_i32_e32 v6, 31, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[14:15]
+; GFX9-NEXT: v_add_u32_e32 v7, 0x80000000, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v4, v5, v4
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_saddsat_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v8
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v9, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v10, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, v3, v11, vcc_lo
-; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v4, v12
-; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v5, v13, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v6, v14, vcc_lo
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v7, v15, vcc_lo
-; GFX10-NEXT: v_xor_b32_e32 v10, v3, v8
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_xor_b32_e32 v11, v7, v9
-; GFX10-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_bfi_b32 v3, v3, 0, v10
-; GFX10-NEXT: v_ashrrev_i32_e32 v10, 31, v8
-; GFX10-NEXT: v_bfi_b32 v7, v7, 0, v11
-; GFX10-NEXT: v_ashrrev_i32_e32 v11, 31, v9
-; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cmp_gt_i32_e64 s4, 0, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v8, -v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v11, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v11, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v11, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v9, -v11, s4
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v0, v8
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v9, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v16, vcc_lo, v2, v10, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v17, vcc_lo, v3, v11, vcc_lo
+; GFX10-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[8:9], v[0:1]
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[16:17], v[2:3]
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[16:17], v[2:3]
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v12, vcc_lo, v4, v12
+; GFX10-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, v5, v13, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v18, vcc_lo, v6, v14, vcc_lo
+; GFX10-NEXT: v_add_co_ci_u32_e32 v19, vcc_lo, v7, v15, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc_lo
+; GFX10-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[12:13], v[4:5]
+; GFX10-NEXT: v_xor_b32_e32 v0, v1, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[18:19], v[6:7]
+; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[14:15]
+; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[18:19], v[6:7]
+; GFX10-NEXT: v_ashrrev_i32_e32 v6, 31, v19
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[14:15]
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v17
+; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x80000000, v6
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc_lo
+; GFX10-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x80000000, v3
+; GFX10-NEXT: v_xor_b32_e32 v2, v2, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc_lo
+; GFX10-NEXT: v_cmp_ne_u16_e64 s4, 0, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v16, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v17, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v12, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v13, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v18, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v19, v7, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_saddsat_v2i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v8
-; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v9, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v10, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e64 v8, null, v3, v11, vcc_lo
-; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, v4, v12
-; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v5, v13, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v6, v14, vcc_lo
-; GFX11-NEXT: v_add_co_ci_u32_e64 v9, null, v7, v15, vcc_lo
-; GFX11-NEXT: v_xor_b32_e32 v10, v3, v8
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX11-NEXT: v_xor_b32_e32 v11, v7, v9
-; GFX11-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX11-NEXT: v_bfi_b32 v3, v3, 0, v10
-; GFX11-NEXT: v_ashrrev_i32_e32 v10, 31, v8
-; GFX11-NEXT: v_bfi_b32 v7, v7, 0, v11
-; GFX11-NEXT: v_ashrrev_i32_e32 v11, 31, v9
-; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v7
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v8, -v10, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v11, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v11, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v11, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v9, -v11, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_saddsat_v2i128:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v8, vcc_lo, v0, v8
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e32 v16, vcc_lo, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v17, null, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[8:9], v[0:1]
+; GFX11-TRUE16-NEXT: v_add_co_u32 v12, s2, v4, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_i64_e64 s0, v[16:17], v[2:3]
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v13, s2, v5, v13, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[10:11]
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v18, s2, v6, v14, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s1, v[16:17], v[2:3]
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v19, null, v7, v15, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, 0, v[10:11]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[12:13], v[4:5]
+; GFX11-TRUE16-NEXT: v_cmp_lt_i64_e64 s2, v[18:19], v[6:7]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e64 s1, 0, v[14:15]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, 0, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, v[18:19], v[6:7]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[14:15]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, 0, 1, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s1
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v17
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v6, 31, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x80000000, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x80000000, v6
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.l, v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v2.l
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v16, v3 :: v_dual_cndmask_b32 v3, v17, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, v12, v6, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v5, v13, v6, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v6, v18, v6, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v7, v19, v7, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_saddsat_v2i128:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v8, vcc_lo, v0, v8
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v16, vcc_lo, v2, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v17, null, v3, v11, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[8:9], v[0:1]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[16:17], v[2:3]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[16:17], v[2:3]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[10:11]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_u32 v12, vcc_lo, v4, v12
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, v5, v13, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e32 v18, vcc_lo, v6, v14, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v19, null, v7, v15, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[10:11]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[12:13], v[4:5]
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[18:19], v[6:7]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[14:15]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[18:19], v[6:7]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v6, 31, v19
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[14:15]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v17
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x80000000, v6
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x80000000, v3
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, v2, v1
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v8, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v2
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v2, v16, v3 :: v_dual_cndmask_b32 v3, v17, v4
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v12, v6, s0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v5, v13, v6, s0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, v18, v6, s0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v7, v19, v7, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
ret <2 x i128> %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sbfx.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sbfx.ll
index 0f8cdbb157f71..b1a36c7eaeedc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sbfx.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sbfx.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - < %s | FileCheck --check-prefixes=GCN %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - < %s | FileCheck --check-prefixes=GCN %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - < %s | FileCheck --check-prefixes=GCN %s
; Test vector signed bitfield extract.
define signext i8 @v_ashr_i8_i32(i32 %value) {
@@ -53,9 +53,9 @@ define i64 @v_ashr_i64(i64 %value) {
; GCN-LABEL: v_ashr_i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_bfe_i32 v2, v0, 10, 4
-; GCN-NEXT: v_bfe_i32 v1, v0, 13, 1
-; GCN-NEXT: v_mov_b32_e32 v0, v2
+; GCN-NEXT: v_ashrrev_i64 v[0:1], 10, v[0:1]
+; GCN-NEXT: v_bfe_i32 v0, v0, 0, 4
+; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%1 = ashr i64 %value, 10
%2 = shl i64 %1, 60
@@ -67,9 +67,9 @@ define i64 @v_lshr_i64(i64 %value) {
; GCN-LABEL: v_lshr_i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_bfe_i32 v2, v0, 10, 4
-; GCN-NEXT: v_bfe_i32 v1, v0, 13, 1
-; GCN-NEXT: v_mov_b32_e32 v0, v2
+; GCN-NEXT: v_ashrrev_i64 v[0:1], 10, v[0:1]
+; GCN-NEXT: v_bfe_i32 v0, v0, 0, 4
+; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%1 = lshr i64 %value, 10
%2 = shl i64 %1, 60
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 2f59daa14bafd..a258949900cdb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -8,150 +8,179 @@ define i64 @v_sdiv_i64(i64 %num, i64 %den) {
; CHECK-LABEL: v_sdiv_i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_or_b32_e32 v4, v1, v3
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: v_mov_b32_e32 v5, v1
+; CHECK-NEXT: v_mov_b32_e32 v4, v0
+; CHECK-NEXT: v_or_b32_e32 v1, v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB0_2
-; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
-; CHECK-NEXT: v_xor_b32_e32 v8, v3, v7
-; CHECK-NEXT: v_xor_b32_e32 v9, v2, v7
-; CHECK-NEXT: v_cvt_f32_u32_e32 v2, v9
-; CHECK-NEXT: v_cvt_f32_u32_e32 v3, v8
-; CHECK-NEXT: v_sub_i32_e32 v12, vcc, 0, v9
-; CHECK-NEXT: v_subb_u32_e32 v13, vcc, 0, v8, vcc
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
-; CHECK-NEXT: v_rcp_f32_e32 v2, v2
-; CHECK-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; CHECK-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
-; CHECK-NEXT: v_trunc_f32_e32 v3, v3
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
-; CHECK-NEXT: v_cvt_u32_f32_e32 v10, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v11, v2
-; CHECK-NEXT: v_mul_lo_u32 v4, v12, v10
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; CHECK-NEXT: v_mul_lo_u32 v5, v13, v11
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v14, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v14, 0
-; CHECK-NEXT: v_mul_hi_u32 v15, v11, v2
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v15, vcc, v15, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v14, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v15, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v11, v2
-; CHECK-NEXT: v_addc_u32_e32 v10, vcc, v10, v3, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; CHECK-NEXT: v_mul_lo_u32 v4, v12, v10
-; CHECK-NEXT: v_mul_lo_u32 v5, v13, v11
-; CHECK-NEXT: v_mul_hi_u32 v13, v11, v2
+; CHECK-NEXT: s_cbranch_execnz .LBB0_3
+; CHECK-NEXT: ; %bb.1: ; %Flow
+; CHECK-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_4
+; CHECK-NEXT: .LBB0_2: ; %.split
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; CHECK-NEXT: .LBB0_3:
+; CHECK-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v2, v0
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v3, v0, vcc
+; CHECK-NEXT: v_xor_b32_e32 v2, v1, v0
+; CHECK-NEXT: v_xor_b32_e32 v1, v3, v0
+; CHECK-NEXT: v_cvt_f32_u32_e32 v3, v2
+; CHECK-NEXT: v_cvt_f32_u32_e32 v6, v1
+; CHECK-NEXT: v_sub_i32_e32 v13, vcc, 0, v2
+; CHECK-NEXT: v_subb_u32_e32 v14, vcc, 0, v1, vcc
+; CHECK-NEXT: v_mac_f32_e32 v3, 0x4f800000, v6
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; CHECK-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
+; CHECK-NEXT: v_mul_f32_e32 v6, 0x2f800000, v3
+; CHECK-NEXT: v_trunc_f32_e32 v6, v6
+; CHECK-NEXT: v_mac_f32_e32 v3, 0xcf800000, v6
+; CHECK-NEXT: v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v12, v6
+; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v13, v3, 0
+; CHECK-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v13, v12, v[7:8]
+; CHECK-NEXT: v_mul_hi_u32 v7, v3, v6
+; CHECK-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v3, v[8:9]
+; CHECK-NEXT: v_mul_lo_u32 v8, v12, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v12, v6
+; CHECK-NEXT: v_mul_lo_u32 v9, v3, v10
+; CHECK-NEXT: v_mul_hi_u32 v11, v3, v10
+; CHECK-NEXT: v_mul_lo_u32 v15, v12, v10
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v11
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v15
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CHECK-NEXT: v_mul_hi_u32 v9, v12, v10
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT: v_addc_u32_e32 v12, vcc, v12, v7, vcc
+; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v13, v3, 0
+; CHECK-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v13, v12, v[7:8]
+; CHECK-NEXT: v_ashrrev_i32_e32 v13, 31, v5
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v13
+; CHECK-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v3, v[8:9]
+; CHECK-NEXT: v_mul_hi_u32 v7, v3, v6
+; CHECK-NEXT: v_xor_b32_e32 v9, v4, v13
+; CHECK-NEXT: v_mul_lo_u32 v4, v12, v6
+; CHECK-NEXT: v_mul_lo_u32 v8, v3, v10
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v12, v6
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v7, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v7, v3, v10
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v8, v4
+; CHECK-NEXT: v_mul_lo_u32 v8, v12, v10
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT: v_mul_hi_u32 v8, v12, v10
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v12, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v12, 0
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v13, vcc, v13, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v12, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v13, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v11, v2
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v10, v3, vcc
-; CHECK-NEXT: v_ashrrev_i32_e32 v5, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_xor_b32_e32 v6, v0, v5
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v1, v5, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
-; CHECK-NEXT: v_mul_hi_u32 v10, v6, v2
-; CHECK-NEXT: v_xor_b32_e32 v11, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v0
-; CHECK-NEXT: v_addc_u32_e32 v12, vcc, 0, v1, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v4, 0
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v12, v3, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v0
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; CHECK-NEXT: v_mul_lo_u32 v4, v9, v3
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v2, 0
-; CHECK-NEXT: v_mul_lo_u32 v10, v8, v2
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v10
-; CHECK-NEXT: v_sub_i32_e32 v4, vcc, v11, v1
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v6, v0
-; CHECK-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v8, vcc
-; CHECK-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v9
-; CHECK-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v9
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v12, v6, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v9, v3
+; CHECK-NEXT: v_mul_lo_u32 v7, v9, v4
+; CHECK-NEXT: v_xor_b32_e32 v10, v5, v13
+; CHECK-NEXT: v_mul_lo_u32 v5, v10, v3
+; CHECK-NEXT: v_mul_hi_u32 v3, v10, v3
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v9, v4
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT: v_mul_lo_u32 v7, v10, v4
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v11, vcc, v3, v5
+; CHECK-NEXT: v_mul_hi_u32 v7, v10, v4
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v2, v11, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_add_i32_e32 v12, vcc, v7, v5
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v2, v12, v[4:5]
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v9, v3
+; CHECK-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v1, v11, v[5:6]
+; CHECK-NEXT: v_subb_u32_e64 v4, s[4:5], v10, v7, vcc
+; CHECK-NEXT: v_sub_i32_e64 v5, s[4:5], v10, v7
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v1
+; CHECK-NEXT: v_subb_u32_e32 v5, vcc, v5, v1, vcc
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v4, v10, v6, s[4:5]
-; CHECK-NEXT: v_add_i32_e64 v6, s[4:5], 2, v2
-; CHECK-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v3, s[4:5]
-; CHECK-NEXT: v_add_i32_e64 v12, s[4:5], 1, v2
-; CHECK-NEXT: v_addc_u32_e64 v13, s[4:5], 0, v3, s[4:5]
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v11, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v1, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v4, v13, v10, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v1, v8
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, v12, v6, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; CHECK-NEXT: v_xor_b32_e32 v2, v5, v7
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
-; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
-; CHECK-NEXT: v_sub_i32_e32 v4, vcc, v1, v2
-; CHECK-NEXT: v_subb_u32_e32 v5, vcc, v0, v2, vcc
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: .LBB0_2: ; %Flow
-; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; CHECK-NEXT: s_cbranch_execz .LBB0_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v3, vcc, 0, v2
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_lo_u32 v3, v3, v1
-; CHECK-NEXT: v_mul_hi_u32 v3, v1, v3
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_mul_lo_u32 v3, v1, v2
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v1
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v2
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v1
+; CHECK-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, 1, v11
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v12, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v5, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v8, v2, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, 1, v6
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v7, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v11, v1, vcc
+; CHECK-NEXT: v_xor_b32_e32 v3, v13, v0
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v1, v3
+; CHECK-NEXT: v_xor_b32_e32 v1, v2, v3
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v0, v2
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v1
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v1, v3, vcc
-; CHECK-NEXT: .LBB0_4: ; %.split
-; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v0, v4
-; CHECK-NEXT: v_mov_b32_e32 v1, v5
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT: s_cbranch_execz .LBB0_2
+; CHECK-NEXT: .LBB0_4:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, v2
+; CHECK-NEXT: v_sub_i32_e32 v1, vcc, 0, v2
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v1, v0
+; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mul_lo_u32 v3, v0, v2
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, 1, v0
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
+; CHECK-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v0
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = sdiv i64 %num, %den
ret i64 %result
@@ -345,342 +374,417 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v5
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v8, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v8
-; GISEL-NEXT: v_xor_b32_e32 v9, v4, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v9
-; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v5
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 0, v9
-; GISEL-NEXT: v_subb_u32_e32 v17, vcc, 0, v5, vcc
-; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v10
-; GISEL-NEXT: v_rcp_f32_e32 v4, v4
-; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GISEL-NEXT: v_mul_f32_e32 v10, 0x2f800000, v4
-; GISEL-NEXT: v_trunc_f32_e32 v10, v10
-; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v10
-; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v10
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v16, v4, 0
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v15
-; GISEL-NEXT: v_mul_lo_u32 v13, v17, v4
-; GISEL-NEXT: v_mul_hi_u32 v19, v4, v10
+; GISEL-NEXT: v_xor_b32_e32 v10, v4, v8
+; GISEL-NEXT: v_xor_b32_e32 v4, v5, v8
+; GISEL-NEXT: v_cvt_f32_u32_e32 v5, v10
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v4
+; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 0, v10
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v4, vcc
+; GISEL-NEXT: v_mac_f32_e32 v5, 0x4f800000, v9
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v5
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT: v_mul_f32_e32 v9, 0x2f800000, v5
+; GISEL-NEXT: v_trunc_f32_e32 v9, v9
+; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v9
+; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v5, 0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v9, v[12:13]
+; GISEL-NEXT: v_mul_hi_u32 v12, v5, v11
+; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v18, v5, v[13:14]
+; GISEL-NEXT: v_mul_lo_u32 v13, v5, v15
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_mul_lo_u32 v13, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v9, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v13, v5, v15
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v14, v12
+; GISEL-NEXT: v_mul_lo_u32 v14, v9, v15
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT: v_mul_hi_u32 v14, v9, v15
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v11, v13
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v18, 0
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v15, v10, 0
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, v19, v11
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v18, 0
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v19, v13
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, v12, v14, vcc
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v10
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, v15, v11, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v16, v4, 0
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v15
-; GISEL-NEXT: v_mul_lo_u32 v13, v17, v4
-; GISEL-NEXT: v_mul_hi_u32 v17, v4, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v14, v12
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; GISEL-NEXT: v_addc_u32_e32 v19, vcc, v9, v12, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v5, 0
+; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v19, v[12:13]
+; GISEL-NEXT: v_mul_hi_u32 v12, v5, v11
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v18, v5, v[13:14]
+; GISEL-NEXT: v_xor_b32_e32 v16, v0, v9
+; GISEL-NEXT: v_mul_lo_u32 v0, v19, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v19, v11
+; GISEL-NEXT: v_mul_lo_u32 v13, v5, v15
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v12, v5, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
+; GISEL-NEXT: v_mul_lo_u32 v13, v19, v15
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v11, v13
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v16, 0
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v15, v10, 0
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v17, v11
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v16, 0
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v17, v13
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, v12, v14, vcc
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v10
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, v15, v11, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v13, 31, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
-; GISEL-NEXT: v_xor_b32_e32 v14, v0, v13
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v1, v13, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v12, 0
-; GISEL-NEXT: v_mul_hi_u32 v15, v14, v4
-; GISEL-NEXT: v_xor_b32_e32 v16, v10, v13
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v16, v4, 0
-; GISEL-NEXT: v_xor_b32_e32 v13, v13, v8
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v15, v0
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v16, v12, 0
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v15, v11, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v4, v0
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v4, v9, v11
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v10, 0
-; GISEL-NEXT: v_mul_lo_u32 v12, v5, v10
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v12
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v16, v1
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v14, v0
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v5, vcc
-; GISEL-NEXT: v_sub_i32_e64 v12, s[4:5], v0, v9
-; GISEL-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v4, v14, v12, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], 2, v10
-; GISEL-NEXT: v_addc_u32_e64 v14, s[4:5], 0, v11, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v15, s[4:5], 1, v10
-; GISEL-NEXT: v_addc_u32_e64 v17, s[4:5], 0, v11, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v16, v1, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v14, v17, v14, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, -1, vcc
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v6, v4
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v7, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v4
-; GISEL-NEXT: v_xor_b32_e32 v17, v17, v4
-; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v17
-; GISEL-NEXT: v_cvt_f32_u32_e32 v18, v6
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v18
-; GISEL-NEXT: v_rcp_f32_e32 v1, v7
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v14, vcc
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_mul_hi_u32 v13, v19, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v5, v0
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v19, v11, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v5
+; GISEL-NEXT: v_xor_b32_e32 v15, v1, v9
+; GISEL-NEXT: v_mul_lo_u32 v1, v15, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v15, v0
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v5
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
+; GISEL-NEXT: v_mul_lo_u32 v12, v15, v5
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v5, v15, v5
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v17, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v5, v11
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v18, v[1:2]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v16, v0
+; GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v7
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v4, v17, v[11:12]
+; GISEL-NEXT: v_subb_u32_e64 v14, s[4:5], v15, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v15, v13
+; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v5
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v7, v5, s[4:5]
+; GISEL-NEXT: v_xor_b32_e32 v7, v6, v5
+; GISEL-NEXT: v_xor_b32_e32 v6, v11, v5
+; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v7
+; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v6
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v10
+; GISEL-NEXT: v_mac_f32_e32 v11, 0x4f800000, v12
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v11, v11
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v10
+; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v11
; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; GISEL-NEXT: v_trunc_f32_e32 v1, v1
; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v7, v15, v12, s[4:5]
-; GISEL-NEXT: v_cvt_u32_f32_e32 v11, v0
-; GISEL-NEXT: v_cvt_u32_f32_e32 v12, v1
-; GISEL-NEXT: v_sub_i32_e64 v14, s[4:5], 0, v17
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, v6, s[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v11, 0
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, v12
-; GISEL-NEXT: v_cndmask_b32_e32 v16, v10, v7, vcc
-; GISEL-NEXT: v_mul_lo_u32 v7, v15, v11
-; GISEL-NEXT: v_mul_hi_u32 v18, v11, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v9
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v1, v7
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v11, v7, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v0, 0
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v7, 0
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v18, v9
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v8, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v0
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v12, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v9, 0
-; GISEL-NEXT: v_mul_lo_u32 v7, v14, v10
-; GISEL-NEXT: v_mul_lo_u32 v8, v15, v9
-; GISEL-NEXT: v_xor_b32_e32 v14, v16, v13
-; GISEL-NEXT: v_mul_hi_u32 v12, v9, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v7
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v1, v8
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v9, v11, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v0, 0
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v7
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v11, 0
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v0
+; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], 0, v7
+; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v1
+; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, v6, s[6:7]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v20, v16, 0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v15, v4
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v19, v[1:2]
+; GISEL-NEXT: v_mul_hi_u32 v1, v16, v0
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v21, v16, v[10:11]
+; GISEL-NEXT: v_mul_lo_u32 v10, v16, v12
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
+; GISEL-NEXT: v_mul_lo_u32 v10, v19, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[6:7]
+; GISEL-NEXT: v_mul_hi_u32 v0, v19, v0
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v15, v4
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v14, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v14, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v17
+; GISEL-NEXT: v_cndmask_b32_e64 v4, v13, v15, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v18, vcc
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, 1, v10
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v13, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[6:7]
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
+; GISEL-NEXT: v_mul_hi_u32 v11, v16, v12
+; GISEL-NEXT: v_ashrrev_i32_e32 v15, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_mul_lo_u32 v11, v19, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v12, v19, v12
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v16, v0
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v12, 0
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v17, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v18, v13, vcc
+; GISEL-NEXT: v_xor_b32_e32 v13, v9, v8
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v20, v14, v[1:2]
+; GISEL-NEXT: v_xor_b32_e32 v1, v4, v13
+; GISEL-NEXT: v_xor_b32_e32 v4, v10, v13
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v15
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v21, v12, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v8, v12, v0
+; GISEL-NEXT: v_xor_b32_e32 v11, v2, v15
+; GISEL-NEXT: v_mul_lo_u32 v2, v14, v0
+; GISEL-NEXT: v_mul_lo_u32 v9, v12, v10
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v15, vcc
+; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v12, v10
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v14, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v15, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v8, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v9, v0
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v10, v1, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v9
-; GISEL-NEXT: v_xor_b32_e32 v10, v0, v9
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v8, 0
-; GISEL-NEXT: v_mul_hi_u32 v11, v10, v7
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v3, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v12, v2, v9
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v7, 0
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v11, v0
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v7, v2
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v2, v0
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v11, v17, v8
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v17, v7, 0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v14, v13
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v5, v13, vcc
-; GISEL-NEXT: v_mul_lo_u32 v5, v6, v7
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v11
-; GISEL-NEXT: v_xor_b32_e32 v4, v9, v4
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT: v_sub_i32_e32 v5, vcc, v12, v3
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v10, v2
-; GISEL-NEXT: v_subb_u32_e64 v5, s[4:5], v5, v6, vcc
-; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], v2, v17
-; GISEL-NEXT: v_subbrev_u32_e64 v5, s[4:5], 0, v5, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v17
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v5, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v5, v11, v10, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], 2, v7
-; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v8, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v13, s[4:5], 1, v7
-; GISEL-NEXT: v_addc_u32_e64 v14, s[4:5], 0, v8, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v12, v3, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v5
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v14, v2, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v11, v0
+; GISEL-NEXT: v_mul_lo_u32 v9, v11, v2
+; GISEL-NEXT: v_xor_b32_e32 v10, v3, v15
+; GISEL-NEXT: v_mul_lo_u32 v3, v10, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v10, v0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v8, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v11, v2
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
+; GISEL-NEXT: v_mul_lo_u32 v9, v10, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v0, v3
+; GISEL-NEXT: v_mul_hi_u32 v9, v10, v2
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v12, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v9, v0
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v7, v14, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v13
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v4, v13, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v12, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v11, v2
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v10, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v10, v3
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v6
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v7
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v6
+; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v4, v8, v9, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, 1, v12
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v14, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v5, v14, v11, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v17
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v7
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, 1, v8
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v9, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v3, v13, v10, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v8, v5, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v2, v4
-; GISEL-NEXT: v_xor_b32_e32 v2, v3, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v8, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
+; GISEL-NEXT: v_xor_b32_e32 v4, v15, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v14, v3, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v4
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v4
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v5, v4, vcc
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_sdiv_v2i64:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_mov_b32_e32 v9, v1
-; CGP-NEXT: v_mov_b32_e32 v8, v0
-; CGP-NEXT: v_or_b32_e32 v0, v9, v5
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT: v_mov_b32_e32 v11, v1
+; CGP-NEXT: v_mov_b32_e32 v10, v0
+; CGP-NEXT: v_or_b32_e32 v1, v11, v5
+; CGP-NEXT: v_mov_b32_e32 v0, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT: v_mov_b32_e32 v8, v2
+; CGP-NEXT: v_mov_b32_e32 v9, v3
; CGP-NEXT: ; implicit-def: $vgpr0_vgpr1
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
; CGP-NEXT: s_cbranch_execz .LBB2_2
; CGP-NEXT: ; %bb.1:
-; CGP-NEXT: v_ashrrev_i32_e32 v10, 31, v5
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v10
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v5, v10, vcc
-; CGP-NEXT: v_xor_b32_e32 v11, v1, v10
-; CGP-NEXT: v_xor_b32_e32 v12, v0, v10
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v12
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v11
-; CGP-NEXT: v_sub_i32_e32 v15, vcc, 0, v12
-; CGP-NEXT: v_subb_u32_e32 v16, vcc, 0, v11, vcc
-; CGP-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CGP-NEXT: v_rcp_f32_e32 v0, v0
-; CGP-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; CGP-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; CGP-NEXT: v_trunc_f32_e32 v1, v1
-; CGP-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; CGP-NEXT: v_cvt_u32_f32_e32 v13, v1
-; CGP-NEXT: v_cvt_u32_f32_e32 v14, v0
-; CGP-NEXT: v_mul_lo_u32 v4, v15, v13
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v14, 0
-; CGP-NEXT: v_mul_lo_u32 v5, v16, v14
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT: v_add_i32_e32 v17, vcc, v1, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v17, 0
-; CGP-NEXT: v_mul_hi_u32 v18, v14, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v0, 0
-; CGP-NEXT: v_add_i32_e32 v18, vcc, v18, v4
-; CGP-NEXT: v_addc_u32_e32 v19, vcc, 0, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v17, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v18, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v19, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v0
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, v13, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v14, 0
-; CGP-NEXT: v_mul_lo_u32 v4, v15, v13
-; CGP-NEXT: v_mul_lo_u32 v5, v16, v14
-; CGP-NEXT: v_mul_hi_u32 v16, v14, v0
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v1, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v15, 0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v0, 0
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v16, v4
-; CGP-NEXT: v_addc_u32_e32 v17, vcc, 0, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v15, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v16, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v17, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v14, v0
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, v13, v1, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v14, 31, v9
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v8, v14
-; CGP-NEXT: v_xor_b32_e32 v8, v0, v14
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v9, v14, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v13, 0
-; CGP-NEXT: v_mul_hi_u32 v9, v8, v4
-; CGP-NEXT: v_xor_b32_e32 v15, v5, v14
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v15, v4, 0
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v0
-; CGP-NEXT: v_addc_u32_e32 v16, vcc, 0, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v13, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v9, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v16, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v0
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; CGP-NEXT: v_mul_lo_u32 v9, v12, v5
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v13, v11, v4
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v9
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v13
-; CGP-NEXT: v_sub_i32_e32 v9, vcc, v15, v1
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v8, v0
-; CGP-NEXT: v_subb_u32_e64 v8, s[4:5], v9, v11, vcc
-; CGP-NEXT: v_sub_i32_e64 v9, s[4:5], v0, v12
-; CGP-NEXT: v_subbrev_u32_e64 v8, s[4:5], 0, v8, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v11
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v12
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v8, v11
-; CGP-NEXT: v_cndmask_b32_e64 v8, v13, v9, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v9, s[4:5], 2, v4
-; CGP-NEXT: v_addc_u32_e64 v13, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v16, s[4:5], 1, v4
-; CGP-NEXT: v_addc_u32_e64 v17, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v15, v1, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v8
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v11
-; CGP-NEXT: v_cndmask_b32_e64 v8, v17, v13, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v12
-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v1, v11
-; CGP-NEXT: v_cndmask_b32_e32 v0, v13, v0, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CGP-NEXT: v_cndmask_b32_e64 v1, v16, v9, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
-; CGP-NEXT: v_xor_b32_e32 v4, v14, v10
-; CGP-NEXT: v_xor_b32_e32 v5, v0, v4
-; CGP-NEXT: v_xor_b32_e32 v0, v1, v4
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CGP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CGP-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v4, v0
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v5, v0, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v1, v0
+; CGP-NEXT: v_xor_b32_e32 v1, v3, v0
+; CGP-NEXT: v_cvt_f32_u32_e32 v3, v2
+; CGP-NEXT: v_cvt_f32_u32_e32 v4, v1
+; CGP-NEXT: v_sub_i32_e32 v16, vcc, 0, v2
+; CGP-NEXT: v_subb_u32_e32 v17, vcc, 0, v1, vcc
+; CGP-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
+; CGP-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
+; CGP-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
+; CGP-NEXT: v_trunc_f32_e32 v4, v4
+; CGP-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
+; CGP-NEXT: v_cvt_u32_f32_e32 v15, v3
+; CGP-NEXT: v_cvt_u32_f32_e32 v14, v4
+; CGP-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v16, v15, 0
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v16, v14, v[4:5]
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v17, v15, v[12:13]
+; CGP-NEXT: v_mul_hi_u32 v5, v15, v3
+; CGP-NEXT: v_mul_lo_u32 v12, v14, v3
+; CGP-NEXT: v_mul_hi_u32 v3, v14, v3
+; CGP-NEXT: v_mul_lo_u32 v13, v15, v4
+; CGP-NEXT: v_mul_hi_u32 v18, v15, v4
+; CGP-NEXT: v_mul_lo_u32 v19, v14, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v14, v4
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v12
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v13, v5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v18
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v19
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v12, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v3
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, v14, v4, vcc
+; CGP-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v16, v15, 0
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v16, v14, v[4:5]
+; CGP-NEXT: v_ashrrev_i32_e32 v16, 31, v11
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v17, v15, v[12:13]
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v16
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, v11, v16, vcc
+; CGP-NEXT: v_mul_hi_u32 v11, v15, v3
+; CGP-NEXT: v_mul_lo_u32 v12, v15, v4
+; CGP-NEXT: v_xor_b32_e32 v13, v5, v16
+; CGP-NEXT: v_mul_lo_u32 v5, v14, v3
+; CGP-NEXT: v_mul_hi_u32 v3, v14, v3
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v11, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v11, v15, v4
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v12, v5
+; CGP-NEXT: v_mul_lo_u32 v12, v14, v4
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_mul_hi_u32 v4, v14, v4
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v11, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v15, v3
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, v14, v4, vcc
+; CGP-NEXT: v_mul_hi_u32 v5, v13, v3
+; CGP-NEXT: v_mul_lo_u32 v11, v13, v4
+; CGP-NEXT: v_xor_b32_e32 v12, v10, v16
+; CGP-NEXT: v_mul_lo_u32 v10, v12, v3
+; CGP-NEXT: v_mul_hi_u32 v3, v12, v3
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v13, v4
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v11, v5
+; CGP-NEXT: v_mul_lo_u32 v11, v12, v4
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v3, v5
+; CGP-NEXT: v_mul_hi_u32 v11, v12, v4
+; CGP-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v2, v14, 0
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v11, v5
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v2, v15, v[4:5]
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v13, v3
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v14, v[10:11]
+; CGP-NEXT: v_subb_u32_e64 v5, s[4:5], v12, v4, vcc
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v12, v4
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v1
+; CGP-NEXT: v_subb_u32_e32 v4, vcc, v4, v1, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v2
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v3, v2
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v5, v1
+; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v5, v10, v11, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v10, vcc, 1, v14
+; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v15, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v1
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
+; CGP-NEXT: v_cndmask_b32_e32 v1, v12, v2, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, 1, v10
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v11, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CGP-NEXT: v_cndmask_b32_e32 v1, v10, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v2, v11, v3, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v1, v14, v1, vcc
+; CGP-NEXT: v_xor_b32_e32 v3, v16, v0
+; CGP-NEXT: v_cndmask_b32_e32 v2, v15, v2, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v1, v3
+; CGP-NEXT: v_xor_b32_e32 v1, v2, v3
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CGP-NEXT: ; implicit-def: $vgpr4
+; CGP-NEXT: ; implicit-def: $vgpr10
; CGP-NEXT: .LBB2_2: ; %Flow1
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
; CGP-NEXT: s_cbranch_execz .LBB2_4
; CGP-NEXT: ; %bb.3:
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v4
@@ -691,164 +795,191 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v1, v1, v0
; CGP-NEXT: v_mul_hi_u32 v1, v0, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_mul_hi_u32 v0, v8, v0
-; CGP-NEXT: v_mul_lo_u32 v1, v0, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v0
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v8, v1
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, v1, v4
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v0
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT: v_mul_hi_u32 v0, v10, v0
; CGP-NEXT: v_mov_b32_e32 v1, 0
+; CGP-NEXT: v_mul_lo_u32 v2, v0, v4
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v0
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v10, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v4
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v0
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; CGP-NEXT: .LBB2_4: ; %.split
-; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_or_b32_e32 v4, v3, v7
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT: v_or_b32_e32 v3, v9, v7
+; CGP-NEXT: v_mov_b32_e32 v2, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; CGP-NEXT: s_cbranch_execz .LBB2_6
-; CGP-NEXT: ; %bb.5:
-; CGP-NEXT: v_ashrrev_i32_e32 v9, 31, v7
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v9
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v7, v9, vcc
-; CGP-NEXT: v_xor_b32_e32 v10, v5, v9
-; CGP-NEXT: v_xor_b32_e32 v11, v4, v9
-; CGP-NEXT: v_cvt_f32_u32_e32 v4, v11
-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v10
-; CGP-NEXT: v_sub_i32_e32 v14, vcc, 0, v11
-; CGP-NEXT: v_subb_u32_e32 v15, vcc, 0, v10, vcc
-; CGP-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CGP-NEXT: v_rcp_f32_e32 v4, v4
-; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CGP-NEXT: v_trunc_f32_e32 v5, v5
-; CGP-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CGP-NEXT: v_cvt_u32_f32_e32 v12, v5
-; CGP-NEXT: v_cvt_u32_f32_e32 v13, v4
-; CGP-NEXT: v_mul_lo_u32 v6, v14, v12
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v7, v15, v13
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v16, 0
-; CGP-NEXT: v_mul_hi_u32 v17, v13, v4
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v4, 0
-; CGP-NEXT: v_add_i32_e32 v17, vcc, v17, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v16, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v17, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v4
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v6, v14, v12
-; CGP-NEXT: v_mul_lo_u32 v7, v15, v13
-; CGP-NEXT: v_mul_hi_u32 v15, v13, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v14, 0
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v4, 0
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v14, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v15, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v13, v4
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v12, v5, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v3, v7, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v6, 0
-; CGP-NEXT: v_mul_hi_u32 v12, v8, v4
-; CGP-NEXT: v_xor_b32_e32 v13, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v4, 0
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v2
-; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v3, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v6, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v14, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v2
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; CGP-NEXT: v_mul_lo_u32 v6, v11, v5
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v12, v10, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
-; CGP-NEXT: v_sub_i32_e32 v6, vcc, v13, v3
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
-; CGP-NEXT: v_subb_u32_e64 v6, s[4:5], v6, v10, vcc
-; CGP-NEXT: v_sub_i32_e64 v8, s[4:5], v2, v11
-; CGP-NEXT: v_subbrev_u32_e64 v6, s[4:5], 0, v6, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v10
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v11
+; CGP-NEXT: s_cbranch_execnz .LBB2_7
+; CGP-NEXT: ; %bb.5: ; %Flow
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT: s_cbranch_execnz .LBB2_8
+; CGP-NEXT: .LBB2_6: ; %.split.split
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT: s_setpc_b64 s[30:31]
+; CGP-NEXT: .LBB2_7:
+; CGP-NEXT: v_ashrrev_i32_e32 v2, 31, v7
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v6, v2
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, v7, v2, vcc
+; CGP-NEXT: v_xor_b32_e32 v4, v3, v2
+; CGP-NEXT: v_xor_b32_e32 v3, v5, v2
+; CGP-NEXT: v_cvt_f32_u32_e32 v5, v4
+; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3
+; CGP-NEXT: v_sub_i32_e32 v14, vcc, 0, v4
+; CGP-NEXT: v_subb_u32_e32 v15, vcc, 0, v3, vcc
+; CGP-NEXT: v_mac_f32_e32 v5, 0x4f800000, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v5, v5
+; CGP-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
+; CGP-NEXT: v_trunc_f32_e32 v6, v6
+; CGP-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
+; CGP-NEXT: v_cvt_u32_f32_e32 v13, v5
+; CGP-NEXT: v_cvt_u32_f32_e32 v12, v6
+; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v13, 0
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v12, v[6:7]
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v13, v[10:11]
+; CGP-NEXT: v_mul_hi_u32 v7, v13, v5
+; CGP-NEXT: v_mul_lo_u32 v10, v12, v5
+; CGP-NEXT: v_mul_hi_u32 v5, v12, v5
+; CGP-NEXT: v_mul_lo_u32 v11, v13, v6
+; CGP-NEXT: v_mul_hi_u32 v16, v13, v6
+; CGP-NEXT: v_mul_lo_u32 v17, v12, v6
+; CGP-NEXT: v_mul_hi_u32 v6, v12, v6
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v16
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v17
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v5
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v6, vcc
+; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v13, 0
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v12, v[6:7]
+; CGP-NEXT: v_ashrrev_i32_e32 v14, 31, v9
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v13, v[10:11]
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v14
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, v9, v14, vcc
+; CGP-NEXT: v_mul_hi_u32 v9, v13, v5
+; CGP-NEXT: v_mul_lo_u32 v10, v13, v6
+; CGP-NEXT: v_xor_b32_e32 v11, v7, v14
+; CGP-NEXT: v_mul_lo_u32 v7, v12, v5
+; CGP-NEXT: v_mul_hi_u32 v5, v12, v5
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v9, v13, v6
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; CGP-NEXT: v_mul_lo_u32 v10, v12, v6
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_mul_hi_u32 v6, v12, v6
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v13, v5
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, v12, v6, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v11, v5
+; CGP-NEXT: v_mul_lo_u32 v9, v11, v6
+; CGP-NEXT: v_xor_b32_e32 v12, v8, v14
+; CGP-NEXT: v_mul_lo_u32 v8, v12, v5
+; CGP-NEXT: v_mul_hi_u32 v5, v12, v5
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v8, v11, v6
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CGP-NEXT: v_mul_lo_u32 v9, v12, v6
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v5, v7
+; CGP-NEXT: v_mul_hi_u32 v9, v12, v6
+; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v4, v13, 0
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v9, v7
+; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v15, v[6:7]
+; CGP-NEXT: v_sub_i32_e32 v5, vcc, v11, v5
+; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v3, v13, v[7:8]
+; CGP-NEXT: v_subb_u32_e64 v6, s[4:5], v12, v9, vcc
+; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v12, v9
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v3
+; CGP-NEXT: v_subb_u32_e32 v7, vcc, v7, v3, vcc
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v10
-; CGP-NEXT: v_cndmask_b32_e64 v6, v12, v8, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v8, s[4:5], 2, v4
-; CGP-NEXT: v_addc_u32_e64 v12, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v14, s[4:5], 1, v4
-; CGP-NEXT: v_addc_u32_e64 v15, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v3, vcc, v13, v3, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v10
-; CGP-NEXT: v_cndmask_b32_e64 v6, v15, v12, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v11
-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v10
-; CGP-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; CGP-NEXT: v_cndmask_b32_e64 v3, v14, v8, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; CGP-NEXT: v_xor_b32_e32 v5, v7, v9
-; CGP-NEXT: v_xor_b32_e32 v3, v3, v5
-; CGP-NEXT: v_xor_b32_e32 v2, v2, v5
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v3, v5
-; CGP-NEXT: v_subb_u32_e32 v5, vcc, v2, v5, vcc
-; CGP-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CGP-NEXT: .LBB2_6: ; %Flow
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; CGP-NEXT: s_cbranch_execz .LBB2_8
-; CGP-NEXT: ; %bb.7:
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v6
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v6
-; CGP-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v4, v3
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v4
+; CGP-NEXT: v_sub_i32_e32 v5, vcc, v5, v4
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v3
+; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v6, v8, v9, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v13
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v15, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v7, v3
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v5, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
+; CGP-NEXT: v_cndmask_b32_e32 v3, v10, v4, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v8
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT: v_cndmask_b32_e32 v3, v8, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v4, v9, v5, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
+; CGP-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
+; CGP-NEXT: v_xor_b32_e32 v5, v14, v2
+; CGP-NEXT: v_cndmask_b32_e32 v4, v15, v4, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v3, v5
+; CGP-NEXT: v_xor_b32_e32 v3, v4, v5
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT: ; implicit-def: $vgpr6
+; CGP-NEXT: ; implicit-def: $vgpr8
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT: s_cbranch_execz .LBB2_6
+; CGP-NEXT: .LBB2_8:
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v6
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v2
+; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v3, v6
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v3
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v6
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v3
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v4, v3, v4, vcc
-; CGP-NEXT: v_mov_b32_e32 v5, 0
-; CGP-NEXT: .LBB2_8: ; %.split.split
-; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_mov_b32_e32 v2, v4
-; CGP-NEXT: v_mov_b32_e32 v3, v5
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v2, v8, v2
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_mul_lo_u32 v4, v2, v6
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v2
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v6
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
+; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v4, v6
+; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v6
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
; CGP-NEXT: s_setpc_b64 s[30:31]
%result = sdiv <2 x i64> %num, %den
ret <2 x i64> %result
@@ -891,73 +1022,672 @@ define i64 @v_sdiv_i64_oddk_denom(i64 %num) {
; CHECK-LABEL: v_sdiv_i64_oddk_denom:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_mov_b32 s6, 0xfd81e19
-; CHECK-NEXT: v_mul_hi_u32 v2, v0, s6
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: s_mov_b32 s7, 0x6ca94220
-; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, s6, v[2:3]
-; CHECK-NEXT: v_mov_b32_e32 v2, v4
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[2:3]
-; CHECK-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v5, v3
-; CHECK-NEXT: v_addc_u32_e64 v3, s[4:5], 0, 0, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
-; CHECK-NEXT: v_mul_lo_u32 v5, v4, s6
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, s6, v[2:3]
-; CHECK-NEXT: v_mul_lo_u32 v2, v4, s7
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v5, v1
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v2, v1
-; CHECK-NEXT: v_ashr_i64 v[2:3], v[0:1], 19
-; CHECK-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; CHECK-NEXT: v_cvt_f32_u32_e32 v3, 0x12d8fb
+; CHECK-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
+; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; CHECK-NEXT: v_xor_b32_e32 v5, v0, v2
+; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; CHECK-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
+; CHECK-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
+; CHECK-NEXT: v_trunc_f32_e32 v4, v4
+; CHECK-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
+; CHECK-NEXT: v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v4
+; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, v3
+; CHECK-NEXT: v_readfirstlane_b32 s4, v3
+; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CHECK-NEXT: v_readfirstlane_b32 s5, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v3, s6
+; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: s_mul_i32 s8, s4, -1
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: v_readfirstlane_b32 s8, v6
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
+; CHECK-NEXT: v_mul_hi_u32 v6, v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v3, v3, s7
+; CHECK-NEXT: s_mul_i32 s10, s5, s6
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s10
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_add_i32 s9, s9, s6
+; CHECK-NEXT: v_readfirstlane_b32 s6, v6
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_mul_i32 s10, s5, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s8
+; CHECK-NEXT: s_cselect_b32 s8, 1, 0
+; CHECK-NEXT: s_add_u32 s6, s6, s10
+; CHECK-NEXT: s_cselect_b32 s10, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s7
+; CHECK-NEXT: s_add_i32 s8, s8, s10
+; CHECK-NEXT: s_add_u32 s6, s6, s9
+; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: s_add_i32 s8, s8, s7
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: s_add_i32 s7, s7, s8
+; CHECK-NEXT: s_add_u32 s4, s4, s6
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, s4
+; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CHECK-NEXT: s_addc_u32 s5, s5, s7
+; CHECK-NEXT: v_xor_b32_e32 v6, v1, v2
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
+; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: s_mul_i32 s8, s4, -1
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s7
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v0
+; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
+; CHECK-NEXT: s_mul_i32 s6, s5, s6
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s6, s8, s6
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_add_i32 s9, s9, s6
+; CHECK-NEXT: v_readfirstlane_b32 s6, v0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_mul_i32 s7, s5, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s8
+; CHECK-NEXT: s_cselect_b32 s8, 1, 0
+; CHECK-NEXT: s_add_u32 s6, s6, s7
+; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v1
+; CHECK-NEXT: s_add_i32 s7, s8, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s9
+; CHECK-NEXT: s_cselect_b32 s8, 1, 0
+; CHECK-NEXT: s_add_i32 s7, s7, s8
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: s_add_i32 s8, s8, s7
+; CHECK-NEXT: s_add_u32 s4, s4, s6
+; CHECK-NEXT: s_addc_u32 s5, s5, s8
+; CHECK-NEXT: v_mul_hi_u32 v0, v5, s4
+; CHECK-NEXT: v_mul_lo_u32 v1, v5, s5
+; CHECK-NEXT: v_mul_lo_u32 v3, v6, s4
+; CHECK-NEXT: v_mul_hi_u32 v4, v5, s5
+; CHECK-NEXT: v_mul_hi_u32 v9, v6, s5
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v3, v6, s4
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v6, s5
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v1, v0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v8, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v3
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v9, v[1:2]
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v5, v0
+; CHECK-NEXT: v_subb_u32_e64 v1, s[4:5], v6, v3, vcc
+; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v7
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v1, -1, v4, s[4:5]
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v8
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v4
+; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v5, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = sdiv i64 %num, 1235195
ret i64 %result
}
define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
-; CHECK-LABEL: v_sdiv_v2i64_oddk_denom:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_mov_b32 s6, 0xfd81e19
-; CHECK-NEXT: v_mul_hi_u32 v4, v0, s6
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: s_mov_b32 s7, 0x6ca94220
-; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, s6, v[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
-; CHECK-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, s7, v[4:5]
-; CHECK-NEXT: v_mul_hi_u32 v4, v2, s6
-; CHECK-NEXT: v_ashrrev_i32_e32 v8, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v9
-; CHECK-NEXT: v_addc_u32_e64 v7, s[4:5], 0, 0, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, s7, v[6:7]
-; CHECK-NEXT: v_mul_lo_u32 v9, v8, s6
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, s6, v[6:7]
-; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, s6, v[4:5]
-; CHECK-NEXT: v_mul_lo_u32 v8, v8, s7
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v1
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
-; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v2, s7, v[4:5]
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v8, v9
-; CHECK-NEXT: v_ashrrev_i32_e32 v8, 31, v3
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v7, v2
-; CHECK-NEXT: v_addc_u32_e64 v5, s[4:5], 0, 0, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, s7, v[4:5]
-; CHECK-NEXT: v_ashr_i64 v[6:7], v[0:1], 19
-; CHECK-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v6, v0
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, s6, v[4:5]
-; CHECK-NEXT: v_mul_lo_u32 v4, v8, s6
-; CHECK-NEXT: v_mul_lo_u32 v5, v8, s7
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
-; CHECK-NEXT: v_ashr_i64 v[4:5], v[2:3], 19
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 31, v3
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_sdiv_v2i64_oddk_denom:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v5, 0
+; GISEL-NEXT: s_mov_b32 s8, 1
+; GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
+; GISEL-NEXT: v_trunc_f32_e32 v6, v6
+; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
+; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v1, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v1, v0, v4
+; GISEL-NEXT: v_mov_b32_e32 v0, 0xffed2705
+; GISEL-NEXT: v_mul_hi_u32 v8, v0, v7
+; GISEL-NEXT: v_readfirstlane_b32 s7, v7
+; GISEL-NEXT: s_mul_i32 s5, s7, 0xffed2705
+; GISEL-NEXT: v_readfirstlane_b32 s6, v5
+; GISEL-NEXT: v_readfirstlane_b32 s9, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v7, s5
+; GISEL-NEXT: s_subb_u32 s4, 0, 0
+; GISEL-NEXT: s_mul_i32 s10, s6, 0xffed2705
+; GISEL-NEXT: s_add_u32 s9, s10, s9
+; GISEL-NEXT: s_mul_i32 s10, s4, s7
+; GISEL-NEXT: s_add_u32 s10, s10, s9
+; GISEL-NEXT: v_readfirstlane_b32 s11, v8
+; GISEL-NEXT: s_mul_i32 s12, s7, s10
+; GISEL-NEXT: v_mul_hi_u32 v8, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, s10
+; GISEL-NEXT: s_mul_i32 s13, s6, s5
+; GISEL-NEXT: s_add_u32 s12, s11, s12
+; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: s_add_u32 s12, s12, s13
+; GISEL-NEXT: s_cselect_b32 s5, 1, 0
+; GISEL-NEXT: s_add_i32 s14, s14, s5
+; GISEL-NEXT: v_readfirstlane_b32 s12, v8
+; GISEL-NEXT: v_readfirstlane_b32 s5, v9
+; GISEL-NEXT: s_mul_i32 s15, s6, s10
+; GISEL-NEXT: s_add_u32 s5, s12, s5
+; GISEL-NEXT: s_cselect_b32 s16, 1, 0
+; GISEL-NEXT: s_add_u32 s5, s5, s15
+; GISEL-NEXT: s_cselect_b32 s15, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v8, v5, s10
+; GISEL-NEXT: s_add_i32 s15, s16, s15
+; GISEL-NEXT: s_add_u32 s5, s5, s14
+; GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; GISEL-NEXT: s_add_i32 s15, s15, s10
+; GISEL-NEXT: v_readfirstlane_b32 s10, v8
+; GISEL-NEXT: s_add_i32 s10, s10, s15
+; GISEL-NEXT: s_add_u32 s5, s7, s5
+; GISEL-NEXT: v_mul_hi_u32 v8, v0, s5
+; GISEL-NEXT: s_mul_i32 s14, s5, 0xffed2705
+; GISEL-NEXT: v_xor_b32_e32 v14, v6, v4
+; GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GISEL-NEXT: s_addc_u32 s10, s6, s10
+; GISEL-NEXT: v_readfirstlane_b32 s15, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, s5, v6
+; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
+; GISEL-NEXT: s_add_u32 s15, s16, s15
+; GISEL-NEXT: s_mul_i32 s4, s4, s5
+; GISEL-NEXT: s_add_u32 s4, s4, s15
+; GISEL-NEXT: v_readfirstlane_b32 s15, v8
+; GISEL-NEXT: v_mov_b32_e32 v8, s4
+; GISEL-NEXT: s_mul_i32 s16, s5, s4
+; GISEL-NEXT: v_mul_hi_u32 v6, s10, v6
+; GISEL-NEXT: v_mul_hi_u32 v9, s5, v8
+; GISEL-NEXT: s_mul_i32 s14, s10, s14
+; GISEL-NEXT: s_add_u32 s15, s15, s16
+; GISEL-NEXT: s_cselect_b32 s16, 1, 0
+; GISEL-NEXT: s_add_u32 s14, s15, s14
+; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: s_add_i32 s16, s16, s14
+; GISEL-NEXT: v_readfirstlane_b32 s14, v6
+; GISEL-NEXT: v_readfirstlane_b32 s15, v9
+; GISEL-NEXT: s_mul_i32 s4, s10, s4
+; GISEL-NEXT: s_add_u32 s14, s14, s15
+; GISEL-NEXT: s_cselect_b32 s15, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s14, s4
+; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v6, s10, v8
+; GISEL-NEXT: s_add_i32 s14, s15, s14
+; GISEL-NEXT: s_add_u32 s4, s4, s16
+; GISEL-NEXT: s_cselect_b32 s15, 1, 0
+; GISEL-NEXT: s_add_i32 s14, s14, s15
+; GISEL-NEXT: v_readfirstlane_b32 s15, v6
+; GISEL-NEXT: s_add_i32 s15, s15, s14
+; GISEL-NEXT: s_add_u32 s4, s5, s4
+; GISEL-NEXT: s_addc_u32 s5, s10, s15
+; GISEL-NEXT: v_mul_hi_u32 v8, v1, s4
+; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, s4
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
+; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v14, s4
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v15, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v12, v10
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v16, v[9:10]
+; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], 0, v15, v[10:11]
+; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], v14, v12, vcc
+; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v14, v12
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, -1, v10, s[4:5]
+; GISEL-NEXT: s_subb_u32 s4, 0, 0
+; GISEL-NEXT: s_mul_i32 s5, s4, s7
+; GISEL-NEXT: s_add_u32 s5, s5, s9
+; GISEL-NEXT: s_mul_i32 s8, s7, s5
+; GISEL-NEXT: v_mul_hi_u32 v7, v7, s5
+; GISEL-NEXT: s_add_u32 s8, s11, s8
+; GISEL-NEXT: s_cselect_b32 s9, 1, 0
+; GISEL-NEXT: s_add_u32 s8, s8, s13
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: s_add_i32 s9, s9, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT: s_mul_i32 s10, s6, s5
+; GISEL-NEXT: s_add_u32 s8, s12, s8
+; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v6
+; GISEL-NEXT: s_cselect_b32 s11, 1, 0
+; GISEL-NEXT: s_add_u32 s8, s8, s10
+; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v5, v5, s5
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v15
+; GISEL-NEXT: s_add_i32 s10, s11, s10
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v16, vcc
+; GISEL-NEXT: s_add_u32 s5, s8, s9
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v6
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9
+; GISEL-NEXT: s_add_i32 s10, s10, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v1, -1, v1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, 1, v10
+; GISEL-NEXT: s_add_i32 s8, s8, s10
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v11, vcc
+; GISEL-NEXT: s_add_u32 s5, s7, s5
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: v_mul_hi_u32 v0, v0, s5
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v10, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v11, v12, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v16, v9, vcc
+; GISEL-NEXT: v_xor_b32_e32 v11, v8, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3
+; GISEL-NEXT: s_mul_i32 s7, s5, 0xffed2705
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v15, v1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT: s_addc_u32 s6, s6, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, s7
+; GISEL-NEXT: v_xor_b32_e32 v5, v2, v8
+; GISEL-NEXT: v_mul_hi_u32 v2, s5, v0
+; GISEL-NEXT: s_mul_i32 s9, s6, 0xffed2705
+; GISEL-NEXT: s_add_u32 s8, s9, s8
+; GISEL-NEXT: s_mul_i32 s4, s4, s5
+; GISEL-NEXT: s_add_u32 s4, s4, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v2
+; GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-NEXT: s_mul_i32 s9, s5, s4
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
+; GISEL-NEXT: v_mul_hi_u32 v7, s5, v2
+; GISEL-NEXT: s_mul_i32 s7, s6, s7
+; GISEL-NEXT: s_add_u32 s8, s8, s9
+; GISEL-NEXT: s_cselect_b32 s9, 1, 0
+; GISEL-NEXT: s_add_u32 s7, s8, s7
+; GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-NEXT: s_add_i32 s9, s9, s7
+; GISEL-NEXT: v_readfirstlane_b32 s7, v0
+; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: s_mul_i32 s4, s6, s4
+; GISEL-NEXT: s_add_u32 s7, s7, s8
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s7, s4
+; GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
+; GISEL-NEXT: s_add_i32 s7, s8, s7
+; GISEL-NEXT: s_add_u32 s4, s4, s9
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: s_add_i32 s7, s7, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GISEL-NEXT: s_add_i32 s8, s8, s7
+; GISEL-NEXT: s_add_u32 s4, s5, s4
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: s_addc_u32 s5, s6, s8
+; GISEL-NEXT: v_mul_hi_u32 v0, v5, s4
+; GISEL-NEXT: v_mul_lo_u32 v2, v5, s5
+; GISEL-NEXT: v_xor_b32_e32 v7, v3, v8
+; GISEL-NEXT: v_mul_lo_u32 v3, v7, s4
+; GISEL-NEXT: v_mul_hi_u32 v9, v5, s5
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v3, v7, s4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GISEL-NEXT: v_mul_lo_u32 v2, v7, s5
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v3
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v2, v0
+; GISEL-NEXT: v_mul_hi_u32 v10, v7, s5
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v12, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v10, v0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v13, v[3:4]
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v12, v[9:10]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v7, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v7, v3
+; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4
+; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v4, -1, v5, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v12
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v13, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, 1, v5
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v6, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v8
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v8
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_v2i64_oddk_denom:
+; CGP: ; %bb.0:
+; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
+; CGP-NEXT: v_cvt_f32_ubyte0_e32 v6, 0
+; CGP-NEXT: v_ashrrev_i32_e32 v5, 31, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT: v_xor_b32_e32 v9, v0, v5
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT: v_trunc_f32_e32 v6, v6
+; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
+; CGP-NEXT: v_cvt_u32_f32_e32 v7, v4
+; CGP-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CGP-NEXT: v_cvt_u32_f32_e32 v0, v6
+; CGP-NEXT: v_xor_b32_e32 v10, v1, v5
+; CGP-NEXT: v_mul_hi_u32 v6, v4, v7
+; CGP-NEXT: v_readfirstlane_b32 s6, v7
+; CGP-NEXT: s_mul_i32 s4, s6, 0xffed2705
+; CGP-NEXT: v_readfirstlane_b32 s7, v0
+; CGP-NEXT: v_readfirstlane_b32 s5, v6
+; CGP-NEXT: v_mul_hi_u32 v6, v7, s4
+; CGP-NEXT: s_mul_i32 s8, s7, 0xffed2705
+; CGP-NEXT: s_add_u32 s5, s8, s5
+; CGP-NEXT: s_mul_i32 s8, s6, -1
+; CGP-NEXT: s_add_u32 s5, s8, s5
+; CGP-NEXT: v_readfirstlane_b32 s8, v6
+; CGP-NEXT: s_mul_i32 s9, s6, s5
+; CGP-NEXT: v_mul_hi_u32 v6, v0, s4
+; CGP-NEXT: v_mul_hi_u32 v7, v7, s5
+; CGP-NEXT: s_mul_i32 s10, s7, s4
+; CGP-NEXT: s_add_u32 s11, s8, s9
+; CGP-NEXT: s_cselect_b32 s12, 1, 0
+; CGP-NEXT: s_add_u32 s11, s11, s10
+; CGP-NEXT: s_cselect_b32 s4, 1, 0
+; CGP-NEXT: s_add_i32 s12, s12, s4
+; CGP-NEXT: v_readfirstlane_b32 s11, v6
+; CGP-NEXT: v_readfirstlane_b32 s13, v7
+; CGP-NEXT: s_mul_i32 s14, s7, s5
+; CGP-NEXT: s_add_u32 s4, s11, s13
+; CGP-NEXT: s_cselect_b32 s15, 1, 0
+; CGP-NEXT: s_add_u32 s4, s4, s14
+; CGP-NEXT: s_cselect_b32 s16, 1, 0
+; CGP-NEXT: v_mul_hi_u32 v0, v0, s5
+; CGP-NEXT: s_add_i32 s15, s15, s16
+; CGP-NEXT: s_add_u32 s4, s4, s12
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
+; CGP-NEXT: s_add_i32 s15, s15, s5
+; CGP-NEXT: v_readfirstlane_b32 s12, v0
+; CGP-NEXT: s_add_i32 s5, s12, s15
+; CGP-NEXT: s_add_u32 s4, s6, s4
+; CGP-NEXT: v_mul_hi_u32 v0, v4, s4
+; CGP-NEXT: s_mul_i32 s15, s4, 0xffed2705
+; CGP-NEXT: s_addc_u32 s5, s7, s5
+; CGP-NEXT: s_mul_i32 s17, s5, 0xffed2705
+; CGP-NEXT: v_readfirstlane_b32 s16, v0
+; CGP-NEXT: v_mov_b32_e32 v0, s15
+; CGP-NEXT: v_mul_hi_u32 v1, s4, v0
+; CGP-NEXT: s_add_u32 s16, s17, s16
+; CGP-NEXT: s_mul_i32 s17, s4, -1
+; CGP-NEXT: s_add_u32 s16, s17, s16
+; CGP-NEXT: v_readfirstlane_b32 s17, v1
+; CGP-NEXT: v_mov_b32_e32 v1, s16
+; CGP-NEXT: s_mul_i32 s18, s4, s16
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v0
+; CGP-NEXT: v_mul_hi_u32 v6, s4, v1
+; CGP-NEXT: s_mul_i32 s15, s5, s15
+; CGP-NEXT: s_add_u32 s17, s17, s18
+; CGP-NEXT: s_cselect_b32 s18, 1, 0
+; CGP-NEXT: s_add_u32 s15, s17, s15
+; CGP-NEXT: s_cselect_b32 s15, 1, 0
+; CGP-NEXT: s_add_i32 s18, s18, s15
+; CGP-NEXT: v_readfirstlane_b32 s15, v0
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
+; CGP-NEXT: s_mul_i32 s16, s5, s16
+; CGP-NEXT: s_add_u32 s15, s15, s17
+; CGP-NEXT: s_cselect_b32 s17, 1, 0
+; CGP-NEXT: s_add_u32 s15, s15, s16
+; CGP-NEXT: s_cselect_b32 s16, 1, 0
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v1
+; CGP-NEXT: s_add_i32 s16, s17, s16
+; CGP-NEXT: s_add_u32 s15, s15, s18
+; CGP-NEXT: s_cselect_b32 s17, 1, 0
+; CGP-NEXT: s_add_i32 s16, s16, s17
+; CGP-NEXT: v_readfirstlane_b32 s17, v0
+; CGP-NEXT: s_add_i32 s17, s17, s16
+; CGP-NEXT: s_add_u32 s4, s4, s15
+; CGP-NEXT: s_addc_u32 s5, s5, s17
+; CGP-NEXT: v_mul_hi_u32 v0, v9, s4
+; CGP-NEXT: v_mul_lo_u32 v1, v9, s5
+; CGP-NEXT: v_mul_lo_u32 v7, v10, s4
+; CGP-NEXT: v_mul_hi_u32 v8, v9, s5
+; CGP-NEXT: v_mov_b32_e32 v6, 0x12d8fb
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v7
+; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v10, s4
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CGP-NEXT: v_mul_lo_u32 v1, v10, s5
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v7, v1
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v1, v0
+; CGP-NEXT: v_mul_hi_u32 v12, v10, s5
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v11, 0
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v7
+; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v6, v12, v[1:2]
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v9, v0
+; CGP-NEXT: v_subb_u32_e64 v1, s[4:5], v10, v7, vcc
+; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v10, v7
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v6
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; CGP-NEXT: v_cndmask_b32_e64 v1, -1, v8, s[4:5]
+; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: s_add_u32 s4, s8, s9
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
+; CGP-NEXT: s_add_u32 s4, s4, s10
+; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: s_cselect_b32 s4, 1, 0
+; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v11
+; CGP-NEXT: s_add_i32 s5, s5, s4
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
+; CGP-NEXT: s_add_u32 s4, s11, s13
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v6
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_u32 s4, s4, s14
+; CGP-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; CGP-NEXT: s_cselect_b32 s9, 1, 0
+; CGP-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v8
+; CGP-NEXT: s_add_i32 s8, s8, s9
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; CGP-NEXT: s_add_u32 s4, s4, s5
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
+; CGP-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CGP-NEXT: s_add_i32 s8, s8, s5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v11, v0, vcc
+; CGP-NEXT: s_add_i32 s12, s12, s8
+; CGP-NEXT: v_cndmask_b32_e32 v1, v12, v7, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
+; CGP-NEXT: s_add_u32 s4, s6, s4
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
+; CGP-NEXT: v_mul_hi_u32 v4, v4, s4
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
+; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
+; CGP-NEXT: v_mov_b32_e32 v2, s6
+; CGP-NEXT: s_addc_u32 s5, s7, s12
+; CGP-NEXT: v_readfirstlane_b32 s7, v4
+; CGP-NEXT: v_mul_hi_u32 v4, s4, v2
+; CGP-NEXT: s_mul_i32 s8, s5, 0xffed2705
+; CGP-NEXT: s_add_u32 s7, s8, s7
+; CGP-NEXT: s_mul_i32 s8, s4, -1
+; CGP-NEXT: s_add_u32 s7, s8, s7
+; CGP-NEXT: v_readfirstlane_b32 s8, v4
+; CGP-NEXT: v_mov_b32_e32 v4, s7
+; CGP-NEXT: s_mul_i32 s9, s4, s7
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
+; CGP-NEXT: v_mul_hi_u32 v5, s4, v4
+; CGP-NEXT: s_mul_i32 s6, s5, s6
+; CGP-NEXT: s_add_u32 s8, s8, s9
+; CGP-NEXT: s_cselect_b32 s9, 1, 0
+; CGP-NEXT: s_add_u32 s6, s8, s6
+; CGP-NEXT: s_cselect_b32 s6, 1, 0
+; CGP-NEXT: s_add_i32 s9, s9, s6
+; CGP-NEXT: v_readfirstlane_b32 s6, v2
+; CGP-NEXT: v_readfirstlane_b32 s8, v5
+; CGP-NEXT: s_mul_i32 s7, s5, s7
+; CGP-NEXT: s_add_u32 s6, s6, s8
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_u32 s6, s6, s7
+; CGP-NEXT: s_cselect_b32 s7, 1, 0
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v4
+; CGP-NEXT: s_add_i32 s7, s8, s7
+; CGP-NEXT: s_add_u32 s6, s6, s9
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_i32 s7, s7, s8
+; CGP-NEXT: v_readfirstlane_b32 s8, v2
+; CGP-NEXT: s_add_i32 s8, s8, s7
+; CGP-NEXT: s_add_u32 s4, s4, s6
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT: s_addc_u32 s5, s5, s8
+; CGP-NEXT: v_mul_hi_u32 v2, v8, s4
+; CGP-NEXT: v_mul_lo_u32 v4, v8, s5
+; CGP-NEXT: v_xor_b32_e32 v9, v3, v7
+; CGP-NEXT: v_mul_lo_u32 v3, v9, s4
+; CGP-NEXT: v_mul_hi_u32 v5, v8, s5
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v3, v9, s4
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2
+; CGP-NEXT: v_mul_lo_u32 v4, v9, s5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v3, v2
+; CGP-NEXT: v_mul_hi_u32 v11, v9, s5
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v10, 0
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v4
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v11, v[3:4]
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
+; CGP-NEXT: v_subb_u32_e64 v3, s[4:5], v9, v4, vcc
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v9, v4
+; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
+; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v3, -1, v5, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v10
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; CGP-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v5
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; CGP-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v11, v4, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v2, v7
+; CGP-NEXT: v_xor_b32_e32 v3, v3, v7
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT: s_setpc_b64 s[30:31]
%result = sdiv <2 x i64> %num, <i64 1235195, i64 1235195>
ret <2 x i64> %result
}
@@ -966,152 +1696,182 @@ define i64 @v_sdiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-LABEL: v_sdiv_i64_pow2_shl_denom:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_mov_b64 s[4:5], 0x1000
-; CHECK-NEXT: v_lshl_b64 v[4:5], s[4:5], v2
-; CHECK-NEXT: v_or_b32_e32 v2, v1, v5
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT: v_mov_b32_e32 v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v4, v1
+; CHECK-NEXT: v_mov_b32_e32 v0, 0x1000
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_lshl_b64 v[5:6], v[0:1], v2
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: v_or_b32_e32 v1, v4, v6
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB7_2
-; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: v_ashrrev_i32_e32 v7, 31, v5
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v7
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v5, v7, vcc
-; CHECK-NEXT: v_xor_b32_e32 v8, v3, v7
-; CHECK-NEXT: v_xor_b32_e32 v9, v2, v7
-; CHECK-NEXT: v_cvt_f32_u32_e32 v2, v9
-; CHECK-NEXT: v_cvt_f32_u32_e32 v3, v8
-; CHECK-NEXT: v_sub_i32_e32 v12, vcc, 0, v9
-; CHECK-NEXT: v_subb_u32_e32 v13, vcc, 0, v8, vcc
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
-; CHECK-NEXT: v_rcp_f32_e32 v2, v2
-; CHECK-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; CHECK-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
-; CHECK-NEXT: v_trunc_f32_e32 v3, v3
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
-; CHECK-NEXT: v_cvt_u32_f32_e32 v10, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v11, v2
-; CHECK-NEXT: v_mul_lo_u32 v4, v12, v10
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; CHECK-NEXT: v_mul_lo_u32 v5, v13, v11
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v14, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v14, 0
-; CHECK-NEXT: v_mul_hi_u32 v15, v11, v2
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v15, vcc, v15, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v14, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v15, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v11, v2
-; CHECK-NEXT: v_addc_u32_e32 v10, vcc, v10, v3, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; CHECK-NEXT: v_mul_lo_u32 v4, v12, v10
-; CHECK-NEXT: v_mul_lo_u32 v5, v13, v11
-; CHECK-NEXT: v_mul_hi_u32 v13, v11, v2
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v12, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v12, 0
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v13, vcc, v13, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v12, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v13, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v11, v2
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v10, v3, vcc
-; CHECK-NEXT: v_ashrrev_i32_e32 v5, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_xor_b32_e32 v6, v0, v5
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v1, v5, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
-; CHECK-NEXT: v_mul_hi_u32 v10, v6, v2
-; CHECK-NEXT: v_xor_b32_e32 v11, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v0
-; CHECK-NEXT: v_addc_u32_e32 v12, vcc, 0, v1, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v4, 0
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v12, v3, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v0
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; CHECK-NEXT: s_cbranch_execnz .LBB7_3
+; CHECK-NEXT: ; %bb.1: ; %Flow
+; CHECK-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT: s_cbranch_execnz .LBB7_4
+; CHECK-NEXT: .LBB7_2: ; %.split
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; CHECK-NEXT: .LBB7_3:
+; CHECK-NEXT: v_ashrrev_i32_e32 v0, 31, v6
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v5, v0
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v6, v0, vcc
+; CHECK-NEXT: v_xor_b32_e32 v2, v1, v0
+; CHECK-NEXT: v_xor_b32_e32 v1, v5, v0
+; CHECK-NEXT: v_cvt_f32_u32_e32 v5, v2
+; CHECK-NEXT: v_cvt_f32_u32_e32 v6, v1
+; CHECK-NEXT: v_sub_i32_e32 v13, vcc, 0, v2
+; CHECK-NEXT: v_subb_u32_e32 v14, vcc, 0, v1, vcc
+; CHECK-NEXT: v_mac_f32_e32 v5, 0x4f800000, v6
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v5, v5
+; CHECK-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CHECK-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
+; CHECK-NEXT: v_trunc_f32_e32 v6, v6
+; CHECK-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
+; CHECK-NEXT: v_cvt_u32_f32_e32 v12, v5
+; CHECK-NEXT: v_cvt_u32_f32_e32 v11, v6
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v12, 0
+; CHECK-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v11, v[6:7]
+; CHECK-NEXT: v_mul_hi_u32 v6, v12, v5
+; CHECK-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v14, v12, v[7:8]
+; CHECK-NEXT: v_mul_lo_u32 v7, v11, v5
+; CHECK-NEXT: v_mul_hi_u32 v5, v11, v5
+; CHECK-NEXT: v_mul_lo_u32 v8, v12, v9
+; CHECK-NEXT: v_mul_hi_u32 v10, v12, v9
+; CHECK-NEXT: v_mul_lo_u32 v15, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v15
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT: v_mul_hi_u32 v8, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT: v_add_i32_e32 v12, vcc, v12, v5
+; CHECK-NEXT: v_addc_u32_e32 v11, vcc, v11, v6, vcc
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v12, 0
+; CHECK-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v11, v[6:7]
+; CHECK-NEXT: v_ashrrev_i32_e32 v13, 31, v4
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v13
+; CHECK-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v14, v12, v[7:8]
+; CHECK-NEXT: v_mul_hi_u32 v6, v12, v5
+; CHECK-NEXT: v_xor_b32_e32 v10, v3, v13
+; CHECK-NEXT: v_mul_lo_u32 v3, v11, v5
+; CHECK-NEXT: v_mul_lo_u32 v7, v12, v9
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v13, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v11, v5
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v12, v9
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
+; CHECK-NEXT: v_mul_lo_u32 v7, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_mul_hi_u32 v7, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v12, v3
+; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v11, v5, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v10, v3
+; CHECK-NEXT: v_mul_lo_u32 v7, v10, v5
+; CHECK-NEXT: v_xor_b32_e32 v9, v4, v13
; CHECK-NEXT: v_mul_lo_u32 v4, v9, v3
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v2, 0
-; CHECK-NEXT: v_mul_lo_u32 v10, v8, v2
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v10
-; CHECK-NEXT: v_sub_i32_e32 v4, vcc, v11, v1
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v6, v0
-; CHECK-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v8, vcc
-; CHECK-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v9
-; CHECK-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v9
+; CHECK-NEXT: v_mul_hi_u32 v3, v9, v3
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v10, v5
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v7, v4
+; CHECK-NEXT: v_mul_lo_u32 v7, v9, v5
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v11, vcc, v3, v4
+; CHECK-NEXT: v_mul_hi_u32 v5, v9, v5
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v2, v11, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v12, vcc, v5, v6
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v2, v12, v[4:5]
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v10, v3
+; CHECK-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v1, v11, v[5:6]
+; CHECK-NEXT: v_subb_u32_e64 v4, s[4:5], v9, v7, vcc
+; CHECK-NEXT: v_sub_i32_e64 v5, s[4:5], v9, v7
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v1
+; CHECK-NEXT: v_subb_u32_e32 v5, vcc, v5, v1, vcc
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v4, v10, v6, s[4:5]
-; CHECK-NEXT: v_add_i32_e64 v6, s[4:5], 2, v2
-; CHECK-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v3, s[4:5]
-; CHECK-NEXT: v_add_i32_e64 v12, s[4:5], 1, v2
-; CHECK-NEXT: v_addc_u32_e64 v13, s[4:5], 0, v3, s[4:5]
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v11, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v1, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v4, v13, v10, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v1, v8
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, v12, v6, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; CHECK-NEXT: v_xor_b32_e32 v3, v5, v7
-; CHECK-NEXT: v_xor_b32_e32 v1, v1, v3
-; CHECK-NEXT: v_xor_b32_e32 v0, v0, v3
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v1, v3
-; CHECK-NEXT: v_subb_u32_e32 v3, vcc, v0, v3, vcc
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: .LBB7_2: ; %Flow
-; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; CHECK-NEXT: s_cbranch_execz .LBB7_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, v4
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, 0, v4
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_lo_u32 v2, v2, v1
-; CHECK-NEXT: v_mul_hi_u32 v2, v1, v2
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v2
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v1
+; CHECK-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, 1, v11
+; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v12, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v5, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v8, v2, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, 1, v6
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v7, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v11, v1, vcc
+; CHECK-NEXT: v_xor_b32_e32 v3, v13, v0
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v1, v3
+; CHECK-NEXT: v_xor_b32_e32 v1, v2, v3
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr3
+; CHECK-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT: s_cbranch_execz .LBB7_2
+; CHECK-NEXT: .LBB7_4:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, v5
+; CHECK-NEXT: v_sub_i32_e32 v1, vcc, 0, v5
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v1, v0
; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_mul_lo_u32 v2, v1, v4
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v1
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v0, v4
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, 1, v1
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CHECK-NEXT: v_cndmask_b32_e32 v2, v1, v2, vcc
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: .LBB7_4: ; %.split
-; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v0, v2
-; CHECK-NEXT: v_mov_b32_e32 v1, v3
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_mul_hi_u32 v0, v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mul_lo_u32 v2, v0, v5
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v0
+; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v5
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v0
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%shl.y = shl i64 4096, %y
%r = sdiv i64 %x, %shl.y
@@ -1122,351 +1882,429 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-LABEL: v_sdiv_v2i64_pow2_shl_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b64 s[4:5], 0x1000
-; GISEL-NEXT: v_lshl_b64 v[7:8], s[4:5], v4
+; GISEL-NEXT: v_mov_b32_e32 v11, 0x1000
+; GISEL-NEXT: v_mov_b32_e32 v12, 0
+; GISEL-NEXT: v_lshl_b64 v[7:8], v[11:12], v4
; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v8
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v4
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v8, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v4
-; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v7
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v4
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, v8, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v8, v5, v4
+; GISEL-NEXT: v_xor_b32_e32 v5, v7, v4
+; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v8
; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v5
-; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v7
-; GISEL-NEXT: v_subb_u32_e32 v16, vcc, 0, v5, vcc
-; GISEL-NEXT: v_mac_f32_e32 v8, 0x4f800000, v9
-; GISEL-NEXT: v_rcp_f32_e32 v8, v8
-; GISEL-NEXT: v_mul_f32_e32 v8, 0x5f7ffffc, v8
-; GISEL-NEXT: v_mul_f32_e32 v9, 0x2f800000, v8
-; GISEL-NEXT: v_trunc_f32_e32 v9, v9
-; GISEL-NEXT: v_mac_f32_e32 v8, 0xcf800000, v9
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v8
-; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v15, v13, 0
-; GISEL-NEXT: v_mul_lo_u32 v10, v15, v14
-; GISEL-NEXT: v_mul_lo_u32 v11, v16, v13
-; GISEL-NEXT: v_mul_hi_u32 v18, v13, v8
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v9, v11
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[6:7], v13, v17, 0
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v14, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v18, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v14, v17, 0
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v18, v11
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v12, vcc
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v8
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v14, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v15, v13, 0
-; GISEL-NEXT: v_mul_lo_u32 v10, v15, v14
-; GISEL-NEXT: v_mul_lo_u32 v11, v16, v13
-; GISEL-NEXT: v_mul_hi_u32 v16, v13, v8
+; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v8
+; GISEL-NEXT: v_subb_u32_e32 v17, vcc, 0, v5, vcc
+; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v9
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v7, v7
+; GISEL-NEXT: v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; GISEL-NEXT: v_mul_f32_e32 v9, 0x2f800000, v7
+; GISEL-NEXT: v_trunc_f32_e32 v13, v9
+; GISEL-NEXT: v_mac_f32_e32 v7, 0xcf800000, v13
+; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v7
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v13
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v7, 0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v15, v16, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v18, v7, v9
+; GISEL-NEXT: v_mul_lo_u32 v19, v16, v9
+; GISEL-NEXT: v_mul_hi_u32 v20, v16, v9
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v7, v[13:14]
+; GISEL-NEXT: v_mul_lo_u32 v10, v7, v9
+; GISEL-NEXT: v_mul_hi_u32 v14, v7, v9
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v18, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v19
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT: v_mul_lo_u32 v13, v16, v9
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v20, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v18, v14
+; GISEL-NEXT: v_mul_hi_u32 v9, v16, v9
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v7, v10
+; GISEL-NEXT: v_addc_u32_e32 v19, vcc, v16, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v18, 0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v15, v19, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v7, v18, v9
+; GISEL-NEXT: v_lshl_b64 v[11:12], v[11:12], v6
+; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v17, v18, v[13:14]
+; GISEL-NEXT: v_mul_lo_u32 v13, v19, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v19, v9
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v12
+; GISEL-NEXT: v_mul_lo_u32 v10, v18, v15
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v13
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v1
+; GISEL-NEXT: v_add_i32_e64 v0, s[4:5], v0, v7
+; GISEL-NEXT: v_xor_b32_e32 v16, v0, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v13, v18, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
+; GISEL-NEXT: v_mul_lo_u32 v10, v19, v15
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v11
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[6:7], v13, v15, 0
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v14, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v14, v15, 0
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v16, v11
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v12, vcc
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v13, v8
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v14, v9, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v11, 31, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
-; GISEL-NEXT: v_xor_b32_e32 v12, v0, v11
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v1, v11, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v12, v10, 0
-; GISEL-NEXT: v_mul_hi_u32 v13, v12, v8
-; GISEL-NEXT: v_xor_b32_e32 v14, v9, v11
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v14, v8, 0
-; GISEL-NEXT: v_xor_b32_e32 v11, v11, v4
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v0
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v14, v10, 0
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v13, v8
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v15, v9, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v8, v0
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v15, v7, v13
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v7, v10, 0
-; GISEL-NEXT: v_lshl_b64 v[8:9], s[4:5], v6
-; GISEL-NEXT: v_mul_lo_u32 v6, v5, v10
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v15
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v6
-; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v14, v1
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_subb_u32_e64 v6, s[4:5], v6, v5, vcc
-; GISEL-NEXT: v_sub_i32_e64 v12, s[4:5], v0, v7
-; GISEL-NEXT: v_subbrev_u32_e64 v6, s[4:5], 0, v6, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v6, v15, v12, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], 2, v10
-; GISEL-NEXT: v_addc_u32_e64 v15, s[4:5], 0, v13, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v16, s[4:5], 1, v10
-; GISEL-NEXT: v_addc_u32_e64 v17, s[4:5], 0, v13, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v14, v1, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v15, v17, v15, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, vcc
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v8, v6
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v9, v6, vcc
-; GISEL-NEXT: v_xor_b32_e32 v8, v8, v6
-; GISEL-NEXT: v_xor_b32_e32 v17, v17, v6
-; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v17
-; GISEL-NEXT: v_cvt_f32_u32_e32 v18, v8
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_mac_f32_e32 v9, 0x4f800000, v18
-; GISEL-NEXT: v_rcp_f32_e32 v1, v9
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v14, v0, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GISEL-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT: v_mul_hi_u32 v13, v19, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v18, v0
+; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], v1, v7, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v19, v9, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v13, v16, v9
+; GISEL-NEXT: v_xor_b32_e32 v15, v1, v7
+; GISEL-NEXT: v_mul_lo_u32 v1, v15, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v15, v0
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v10, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v16, v9
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v13, v1
+; GISEL-NEXT: v_mul_lo_u32 v13, v15, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v17, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v9, v10
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v8, v18, v[1:2]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v16, v0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v5, v17, v[9:10]
+; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], v15, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v15, v13
+; GISEL-NEXT: v_add_i32_e64 v9, s[4:5], v11, v6
+; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v12, v6, s[4:5]
+; GISEL-NEXT: v_xor_b32_e32 v10, v9, v6
+; GISEL-NEXT: v_xor_b32_e32 v9, v11, v6
+; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v10
+; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v9
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v8
+; GISEL-NEXT: v_mac_f32_e32 v11, 0x4f800000, v12
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v11, v11
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8
+; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v11
; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; GISEL-NEXT: v_trunc_f32_e32 v1, v1
; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v5, v16, v12, s[4:5]
-; GISEL-NEXT: v_cvt_u32_f32_e32 v12, v0
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v1
-; GISEL-NEXT: v_sub_i32_e64 v14, s[4:5], 0, v17
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, v8, s[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v12, 0
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, v13
-; GISEL-NEXT: v_cndmask_b32_e32 v16, v10, v5, vcc
-; GISEL-NEXT: v_mul_lo_u32 v5, v15, v12
-; GISEL-NEXT: v_mul_hi_u32 v18, v12, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v9
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v1, v5
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v12, v5, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v0, 0
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v5, 0
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v11
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v18, v9
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v12, v0
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v13, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v9, 0
-; GISEL-NEXT: v_mul_lo_u32 v4, v14, v10
-; GISEL-NEXT: v_mul_lo_u32 v5, v15, v9
-; GISEL-NEXT: v_xor_b32_e32 v14, v16, v11
-; GISEL-NEXT: v_mul_hi_u32 v13, v9, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v1, v5
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v9, v12, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v0, 0
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v4
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v5, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v15, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GISEL-NEXT: v_cvt_u32_f32_e32 v8, v0
+; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], 0, v10
+; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v1
+; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, v9, s[6:7]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v20, v8, 0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v15, v5
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v20, v19, v[1:2]
+; GISEL-NEXT: v_mul_hi_u32 v1, v8, v0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[6:7], v21, v8, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v11, v8, v13
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v11
+; GISEL-NEXT: v_mul_lo_u32 v11, v19, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT: v_mul_hi_u32 v0, v19, v0
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v15, v5
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v16, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v17
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v14, v15, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v18, vcc
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, 1, v11
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v14, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[6:7]
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v14, v14, v16, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
+; GISEL-NEXT: v_mul_hi_u32 v12, v8, v13
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; GISEL-NEXT: v_mul_lo_u32 v12, v19, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v13, v19, v13
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v15, v12
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v13, v1
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v8, v0
+; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v19, v1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v12, 0
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v17, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v18, v14, vcc
+; GISEL-NEXT: v_xor_b32_e32 v14, v7, v4
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v20, v13, v[1:2]
+; GISEL-NEXT: v_xor_b32_e32 v1, v8, v14
+; GISEL-NEXT: v_ashrrev_i32_e32 v15, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v15
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v21, v12, v[4:5]
+; GISEL-NEXT: v_mul_hi_u32 v4, v12, v0
+; GISEL-NEXT: v_xor_b32_e32 v16, v2, v15
+; GISEL-NEXT: v_mul_lo_u32 v2, v13, v0
+; GISEL-NEXT: v_mul_lo_u32 v5, v12, v7
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v15, vcc
+; GISEL-NEXT: v_mul_hi_u32 v0, v13, v0
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v4, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v4, v12, v7
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v5, v2
+; GISEL-NEXT: v_mul_lo_u32 v5, v13, v7
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v9, v0
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v10, v1, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v9
-; GISEL-NEXT: v_xor_b32_e32 v10, v0, v9
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v5, 0
-; GISEL-NEXT: v_mul_hi_u32 v12, v10, v4
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v3, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v13, v2, v9
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v4, 0
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v12, v0
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v5, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_mul_hi_u32 v5, v13, v7
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v12, v3, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v2, v0
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v12, v17, v5
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v17, v4, 0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v14, v11
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v7, v11, vcc
-; GISEL-NEXT: v_mul_lo_u32 v7, v8, v4
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v12
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT: v_sub_i32_e32 v7, vcc, v13, v3
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v10, v2
-; GISEL-NEXT: v_subb_u32_e64 v7, s[4:5], v7, v8, vcc
-; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], v2, v17
-; GISEL-NEXT: v_subbrev_u32_e64 v7, s[4:5], 0, v7, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v17
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v7, v11, v10, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], 2, v4
-; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v5, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], 1, v4
-; GISEL-NEXT: v_addc_u32_e64 v14, s[4:5], 0, v5, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v13, v3, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v3, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v7, v14, v11, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v17
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v5, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v13, v2, vcc
+; GISEL-NEXT: v_mul_hi_u32 v4, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v5, v16, v2
+; GISEL-NEXT: v_xor_b32_e32 v12, v3, v15
+; GISEL-NEXT: v_mul_lo_u32 v3, v12, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v12, v0
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v4, v16, v2
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; GISEL-NEXT: v_mul_lo_u32 v5, v12, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
+; GISEL-NEXT: v_mul_hi_u32 v5, v12, v2
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v13, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, v5, v0
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v17, v[3:4]
+; GISEL-NEXT: v_xor_b32_e32 v11, v11, v14
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v14
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v9, v13, v[4:5]
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v14, vcc
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v16, v2
+; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], v12, v7, vcc
+; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v12, v7
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v9
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v10
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v3, v9
+; GISEL-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v3, v5, v7, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v13
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v17, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v4, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v10
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v8
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v4, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, 1, v5
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v7, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v3, v12, v10, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v4, v9, v6
-; GISEL-NEXT: v_xor_b32_e32 v5, v2, v4
-; GISEL-NEXT: v_xor_b32_e32 v2, v3, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v13, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v17, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v4, v15, v6
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v4
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v4
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v5, v4, vcc
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_sdiv_v2i64_pow2_shl_denom:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: s_mov_b64 s[4:5], 0x1000
-; CGP-NEXT: v_lshl_b64 v[11:12], s[4:5], v4
-; CGP-NEXT: v_mov_b32_e32 v5, v1
-; CGP-NEXT: v_mov_b32_e32 v7, v0
-; CGP-NEXT: v_lshl_b64 v[9:10], s[4:5], v6
-; CGP-NEXT: v_or_b32_e32 v0, v5, v12
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT: v_mov_b32_e32 v5, v2
+; CGP-NEXT: v_mov_b32_e32 v7, v3
+; CGP-NEXT: v_mov_b32_e32 v2, 0x1000
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_lshl_b64 v[11:12], v[2:3], v4
+; CGP-NEXT: v_mov_b32_e32 v9, v1
+; CGP-NEXT: v_mov_b32_e32 v8, v0
+; CGP-NEXT: v_or_b32_e32 v1, v9, v12
+; CGP-NEXT: v_mov_b32_e32 v0, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; CGP-NEXT: ; implicit-def: $vgpr0_vgpr1
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
; CGP-NEXT: s_cbranch_execz .LBB8_2
; CGP-NEXT: ; %bb.1:
-; CGP-NEXT: v_ashrrev_i32_e32 v6, 31, v12
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v11, v6
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v12, v6, vcc
-; CGP-NEXT: v_xor_b32_e32 v8, v1, v6
-; CGP-NEXT: v_xor_b32_e32 v13, v0, v6
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v13
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v8
-; CGP-NEXT: v_sub_i32_e32 v15, vcc, 0, v13
-; CGP-NEXT: v_subb_u32_e32 v16, vcc, 0, v8, vcc
-; CGP-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CGP-NEXT: v_rcp_f32_e32 v0, v0
-; CGP-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; CGP-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; CGP-NEXT: v_trunc_f32_e32 v1, v1
-; CGP-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; CGP-NEXT: v_cvt_u32_f32_e32 v4, v1
-; CGP-NEXT: v_cvt_u32_f32_e32 v14, v0
-; CGP-NEXT: v_mul_lo_u32 v11, v15, v4
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v14, 0
+; CGP-NEXT: v_ashrrev_i32_e32 v0, 31, v12
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v11, v0
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, v12, v0, vcc
+; CGP-NEXT: v_xor_b32_e32 v4, v1, v0
+; CGP-NEXT: v_xor_b32_e32 v1, v10, v0
+; CGP-NEXT: v_cvt_f32_u32_e32 v10, v4
+; CGP-NEXT: v_cvt_f32_u32_e32 v11, v1
+; CGP-NEXT: v_sub_i32_e32 v18, vcc, 0, v4
+; CGP-NEXT: v_subb_u32_e32 v19, vcc, 0, v1, vcc
+; CGP-NEXT: v_mac_f32_e32 v10, 0x4f800000, v11
+; CGP-NEXT: v_rcp_iflag_f32_e32 v10, v10
+; CGP-NEXT: v_mul_f32_e32 v10, 0x5f7ffffc, v10
+; CGP-NEXT: v_mul_f32_e32 v11, 0x2f800000, v10
+; CGP-NEXT: v_trunc_f32_e32 v11, v11
+; CGP-NEXT: v_mac_f32_e32 v10, 0xcf800000, v11
+; CGP-NEXT: v_cvt_u32_f32_e32 v17, v10
+; CGP-NEXT: v_cvt_u32_f32_e32 v16, v11
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v18, v17, 0
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v18, v16, v[11:12]
+; CGP-NEXT: v_mul_hi_u32 v11, v17, v10
+; CGP-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v19, v17, v[12:13]
+; CGP-NEXT: v_mul_lo_u32 v12, v16, v10
+; CGP-NEXT: v_mul_hi_u32 v10, v16, v10
+; CGP-NEXT: v_mul_lo_u32 v13, v17, v14
+; CGP-NEXT: v_mul_hi_u32 v15, v17, v14
+; CGP-NEXT: v_mul_lo_u32 v20, v16, v14
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v15
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v20
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_mul_hi_u32 v13, v16, v14
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_add_i32_e32 v17, vcc, v17, v10
+; CGP-NEXT: v_addc_u32_e32 v16, vcc, v16, v11, vcc
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v18, v17, 0
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v18, v16, v[11:12]
+; CGP-NEXT: v_ashrrev_i32_e32 v18, 31, v9
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v18
+; CGP-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v19, v17, v[12:13]
+; CGP-NEXT: v_mul_hi_u32 v11, v17, v10
+; CGP-NEXT: v_xor_b32_e32 v15, v8, v18
+; CGP-NEXT: v_mul_lo_u32 v8, v16, v10
+; CGP-NEXT: v_mul_lo_u32 v12, v17, v14
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, v9, v18, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v16, v10
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v11, v17, v14
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v12, v8
; CGP-NEXT: v_mul_lo_u32 v12, v16, v14
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v11
-; CGP-NEXT: v_add_i32_e32 v17, vcc, v1, v12
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v14, v17, 0
-; CGP-NEXT: v_mul_hi_u32 v18, v14, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0
-; CGP-NEXT: v_add_i32_e32 v18, vcc, v18, v11
-; CGP-NEXT: v_addc_u32_e32 v19, vcc, 0, v12, vcc
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v17, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v18, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v19, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v12, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v11
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v0
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v4, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v14, 0
-; CGP-NEXT: v_mul_lo_u32 v11, v15, v4
-; CGP-NEXT: v_mul_lo_u32 v12, v16, v14
-; CGP-NEXT: v_mul_hi_u32 v16, v14, v0
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v11
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v1, v12
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v14, v15, 0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v16, v11
-; CGP-NEXT: v_addc_u32_e32 v17, vcc, 0, v12, vcc
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v15, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v16, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v17, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v12, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v11
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v14, v0
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, v4, v1, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v14, 31, v5
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v7, v14
-; CGP-NEXT: v_xor_b32_e32 v7, v0, v14
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v12, 0
-; CGP-NEXT: v_mul_hi_u32 v15, v7, v11
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v5, v14, vcc
-; CGP-NEXT: v_xor_b32_e32 v16, v4, v14
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v11, 0
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v15, v0
-; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v16, v12, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v11, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v15, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v0
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; CGP-NEXT: v_mul_lo_u32 v11, v13, v5
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v12, v8, v4
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v11
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v12
-; CGP-NEXT: v_sub_i32_e32 v11, vcc, v16, v1
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v7, v0
-; CGP-NEXT: v_subb_u32_e64 v7, s[4:5], v11, v8, vcc
-; CGP-NEXT: v_sub_i32_e64 v11, s[4:5], v0, v13
-; CGP-NEXT: v_subbrev_u32_e64 v7, s[4:5], 0, v7, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v8
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v13
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_mul_hi_u32 v12, v16, v14
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v17, v8
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, v16, v10, vcc
+; CGP-NEXT: v_mul_hi_u32 v11, v15, v8
+; CGP-NEXT: v_mul_lo_u32 v12, v15, v10
+; CGP-NEXT: v_xor_b32_e32 v14, v9, v18
+; CGP-NEXT: v_mul_lo_u32 v9, v14, v8
+; CGP-NEXT: v_mul_hi_u32 v8, v14, v8
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v11, v15, v10
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v12, v9
+; CGP-NEXT: v_mul_lo_u32 v12, v14, v10
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_add_i32_e32 v16, vcc, v8, v9
+; CGP-NEXT: v_mul_hi_u32 v10, v14, v10
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v4, v16, 0
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_add_i32_e32 v17, vcc, v10, v11
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v4, v17, v[9:10]
+; CGP-NEXT: v_sub_i32_e32 v8, vcc, v15, v8
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v1, v16, v[10:11]
+; CGP-NEXT: v_subb_u32_e64 v9, s[4:5], v14, v12, vcc
+; CGP-NEXT: v_sub_i32_e64 v10, s[4:5], v14, v12
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v1
+; CGP-NEXT: v_subb_u32_e32 v10, vcc, v10, v1, vcc
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v8
-; CGP-NEXT: v_cndmask_b32_e64 v7, v12, v11, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v11, s[4:5], 2, v4
-; CGP-NEXT: v_addc_u32_e64 v12, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v15, s[4:5], 1, v4
-; CGP-NEXT: v_addc_u32_e64 v17, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v16, v1, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v8
-; CGP-NEXT: v_cndmask_b32_e64 v7, v17, v12, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v13
-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v1, v8
-; CGP-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CGP-NEXT: v_cndmask_b32_e64 v1, v15, v11, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
-; CGP-NEXT: v_xor_b32_e32 v4, v14, v6
-; CGP-NEXT: v_xor_b32_e32 v5, v0, v4
-; CGP-NEXT: v_xor_b32_e32 v0, v1, v4
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v4
+; CGP-NEXT: v_sub_i32_e32 v8, vcc, v8, v4
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v1
+; CGP-NEXT: v_subbrev_u32_e32 v10, vcc, 0, v10, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v9, v11, v12, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v11, vcc, 1, v16
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v17, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v10, v1
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v8, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v10, v1
+; CGP-NEXT: v_cndmask_b32_e32 v1, v13, v4, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v11
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v12, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CGP-NEXT: v_cndmask_b32_e32 v1, v11, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v4, v12, v8, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; CGP-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc
+; CGP-NEXT: v_xor_b32_e32 v8, v18, v0
+; CGP-NEXT: v_cndmask_b32_e32 v4, v17, v4, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v1, v8
+; CGP-NEXT: v_xor_b32_e32 v1, v4, v8
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v8, vcc
; CGP-NEXT: ; implicit-def: $vgpr11_vgpr12
-; CGP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CGP-NEXT: ; implicit-def: $vgpr8
; CGP-NEXT: .LBB8_2: ; %Flow1
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT: v_lshl_b64 v[9:10], v[2:3], v6
+; CGP-NEXT: s_xor_b64 exec, exec, s[6:7]
; CGP-NEXT: s_cbranch_execz .LBB8_4
; CGP-NEXT: ; %bb.3:
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v11
@@ -1477,164 +2315,191 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v1, v1, v0
; CGP-NEXT: v_mul_hi_u32 v1, v0, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_mul_hi_u32 v0, v7, v0
-; CGP-NEXT: v_mul_lo_u32 v1, v0, v11
-; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v0
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v7, v1
-; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v11
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v11
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v0
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v11
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT: v_mul_hi_u32 v0, v8, v0
; CGP-NEXT: v_mov_b32_e32 v1, 0
+; CGP-NEXT: v_mul_lo_u32 v2, v0, v11
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v0
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v11
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v11
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v0
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v11
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; CGP-NEXT: .LBB8_4: ; %.split
-; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_or_b32_e32 v4, v3, v10
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT: v_or_b32_e32 v3, v7, v10
+; CGP-NEXT: v_mov_b32_e32 v2, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; CGP-NEXT: s_cbranch_execz .LBB8_6
-; CGP-NEXT: ; %bb.5:
-; CGP-NEXT: v_ashrrev_i32_e32 v11, 31, v10
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v9, v11
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v10, v11, vcc
-; CGP-NEXT: v_xor_b32_e32 v9, v5, v11
-; CGP-NEXT: v_xor_b32_e32 v10, v4, v11
-; CGP-NEXT: v_cvt_f32_u32_e32 v4, v10
-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v9
-; CGP-NEXT: v_sub_i32_e32 v14, vcc, 0, v10
-; CGP-NEXT: v_subb_u32_e32 v15, vcc, 0, v9, vcc
-; CGP-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CGP-NEXT: v_rcp_f32_e32 v4, v4
-; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CGP-NEXT: v_trunc_f32_e32 v5, v5
-; CGP-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CGP-NEXT: v_cvt_u32_f32_e32 v12, v5
-; CGP-NEXT: v_cvt_u32_f32_e32 v13, v4
-; CGP-NEXT: v_mul_lo_u32 v6, v14, v12
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v7, v15, v13
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v16, 0
-; CGP-NEXT: v_mul_hi_u32 v17, v13, v4
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v4, 0
-; CGP-NEXT: v_add_i32_e32 v17, vcc, v17, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v16, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v17, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v4
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v6, v14, v12
-; CGP-NEXT: v_mul_lo_u32 v7, v15, v13
-; CGP-NEXT: v_mul_hi_u32 v15, v13, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v14, 0
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v4, 0
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v14, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v15, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v13, v4
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v12, v5, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v3, v7, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v6, 0
-; CGP-NEXT: v_mul_hi_u32 v12, v8, v4
-; CGP-NEXT: v_xor_b32_e32 v13, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v4, 0
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v2
-; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v3, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v6, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v14, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v2
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; CGP-NEXT: v_mul_lo_u32 v6, v10, v5
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v12, v9, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
-; CGP-NEXT: v_sub_i32_e32 v6, vcc, v13, v3
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
-; CGP-NEXT: v_subb_u32_e64 v6, s[4:5], v6, v9, vcc
-; CGP-NEXT: v_sub_i32_e64 v8, s[4:5], v2, v10
-; CGP-NEXT: v_subbrev_u32_e64 v6, s[4:5], 0, v6, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v9
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v10
+; CGP-NEXT: s_cbranch_execnz .LBB8_7
+; CGP-NEXT: ; %bb.5: ; %Flow
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT: s_cbranch_execnz .LBB8_8
+; CGP-NEXT: .LBB8_6: ; %.split.split
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT: s_setpc_b64 s[30:31]
+; CGP-NEXT: .LBB8_7:
+; CGP-NEXT: v_ashrrev_i32_e32 v2, 31, v10
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v9, v2
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, v10, v2, vcc
+; CGP-NEXT: v_xor_b32_e32 v4, v3, v2
+; CGP-NEXT: v_xor_b32_e32 v3, v6, v2
+; CGP-NEXT: v_cvt_f32_u32_e32 v6, v4
+; CGP-NEXT: v_cvt_f32_u32_e32 v8, v3
+; CGP-NEXT: v_sub_i32_e32 v15, vcc, 0, v4
+; CGP-NEXT: v_subb_u32_e32 v16, vcc, 0, v3, vcc
+; CGP-NEXT: v_mac_f32_e32 v6, 0x4f800000, v8
+; CGP-NEXT: v_rcp_iflag_f32_e32 v6, v6
+; CGP-NEXT: v_mul_f32_e32 v6, 0x5f7ffffc, v6
+; CGP-NEXT: v_mul_f32_e32 v8, 0x2f800000, v6
+; CGP-NEXT: v_trunc_f32_e32 v8, v8
+; CGP-NEXT: v_mac_f32_e32 v6, 0xcf800000, v8
+; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT: v_cvt_u32_f32_e32 v14, v8
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v6, 0
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v14, v[9:10]
+; CGP-NEXT: v_mul_hi_u32 v9, v6, v8
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v16, v6, v[10:11]
+; CGP-NEXT: v_mul_lo_u32 v10, v14, v8
+; CGP-NEXT: v_mul_hi_u32 v8, v14, v8
+; CGP-NEXT: v_mul_lo_u32 v11, v6, v12
+; CGP-NEXT: v_mul_hi_u32 v13, v6, v12
+; CGP-NEXT: v_mul_lo_u32 v17, v14, v12
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v13
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v17
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_mul_hi_u32 v11, v14, v12
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, v14, v9, vcc
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v6, 0
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v14, v[9:10]
+; CGP-NEXT: v_ashrrev_i32_e32 v15, 31, v7
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v15
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v16, v6, v[10:11]
+; CGP-NEXT: v_mul_hi_u32 v9, v6, v8
+; CGP-NEXT: v_xor_b32_e32 v11, v5, v15
+; CGP-NEXT: v_mul_lo_u32 v5, v14, v8
+; CGP-NEXT: v_mul_lo_u32 v10, v6, v12
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, v7, v15, vcc
+; CGP-NEXT: v_mul_hi_u32 v8, v14, v8
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v9, v6, v12
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT: v_mul_lo_u32 v10, v14, v12
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_mul_hi_u32 v10, v14, v12
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, v14, v8, vcc
+; CGP-NEXT: v_mul_hi_u32 v8, v11, v5
+; CGP-NEXT: v_mul_lo_u32 v9, v11, v6
+; CGP-NEXT: v_xor_b32_e32 v12, v7, v15
+; CGP-NEXT: v_mul_lo_u32 v7, v12, v5
+; CGP-NEXT: v_mul_hi_u32 v5, v12, v5
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v8, v11, v6
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; CGP-NEXT: v_mul_lo_u32 v9, v12, v6
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v5, v7
+; CGP-NEXT: v_mul_hi_u32 v9, v12, v6
+; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v4, v13, 0
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v9, v7
+; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v14, v[6:7]
+; CGP-NEXT: v_sub_i32_e32 v5, vcc, v11, v5
+; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v3, v13, v[7:8]
+; CGP-NEXT: v_subb_u32_e64 v6, s[4:5], v12, v9, vcc
+; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v12, v9
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v3
+; CGP-NEXT: v_subb_u32_e32 v7, vcc, v7, v3, vcc
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v9
-; CGP-NEXT: v_cndmask_b32_e64 v6, v12, v8, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v8, s[4:5], 2, v4
-; CGP-NEXT: v_addc_u32_e64 v12, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v14, s[4:5], 1, v4
-; CGP-NEXT: v_addc_u32_e64 v15, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v3, vcc, v13, v3, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v9
-; CGP-NEXT: v_cndmask_b32_e64 v6, v15, v12, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v10
-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v9
-; CGP-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; CGP-NEXT: v_cndmask_b32_e64 v3, v14, v8, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; CGP-NEXT: v_xor_b32_e32 v5, v7, v11
-; CGP-NEXT: v_xor_b32_e32 v3, v3, v5
-; CGP-NEXT: v_xor_b32_e32 v2, v2, v5
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v3, v5
-; CGP-NEXT: v_subb_u32_e32 v5, vcc, v2, v5, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v4
+; CGP-NEXT: v_sub_i32_e32 v5, vcc, v5, v4
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v3
+; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v6, v8, v9, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v13
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v14, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v7, v3
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v5, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
+; CGP-NEXT: v_cndmask_b32_e32 v3, v10, v4, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v8
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT: v_cndmask_b32_e32 v3, v8, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v4, v9, v5, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
+; CGP-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
+; CGP-NEXT: v_xor_b32_e32 v5, v15, v2
+; CGP-NEXT: v_cndmask_b32_e32 v4, v14, v4, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v3, v5
+; CGP-NEXT: v_xor_b32_e32 v3, v4, v5
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v5, vcc
; CGP-NEXT: ; implicit-def: $vgpr9_vgpr10
-; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CGP-NEXT: .LBB8_6: ; %Flow
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; CGP-NEXT: s_cbranch_execz .LBB8_8
-; CGP-NEXT: ; %bb.7:
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v9
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v9
-; CGP-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v4, v3
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: ; implicit-def: $vgpr5
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT: s_cbranch_execz .LBB8_6
+; CGP-NEXT: .LBB8_8:
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v9
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v9
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v2
+; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v3, v9
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v3
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v9
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v9
-; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v3
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v9
-; CGP-NEXT: v_cndmask_b32_e32 v4, v3, v4, vcc
-; CGP-NEXT: v_mov_b32_e32 v5, 0
-; CGP-NEXT: .LBB8_8: ; %.split.split
-; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_mov_b32_e32 v2, v4
-; CGP-NEXT: v_mov_b32_e32 v3, v5
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v2, v5, v2
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_mul_lo_u32 v4, v2, v9
+; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v2
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v9
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
+; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v4, v9
+; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v9
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
; CGP-NEXT: s_setpc_b64 s[30:31]
%shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
%r = sdiv <2 x i64> %x, %shl.y
@@ -1917,50 +2782,50 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; CGP-LABEL: v_sdiv_v2i64_24bit:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v1
+; CGP-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; CGP-NEXT: v_cvt_f32_u32_e32 v1, v4
; CGP-NEXT: v_and_b32_e32 v5, 0xffffff, v6
-; CGP-NEXT: v_cvt_f32_u32_e32 v7, v5
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, 0, v1
-; CGP-NEXT: v_rcp_f32_e32 v3, v3
-; CGP-NEXT: v_rcp_f32_e32 v7, v7
-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
-; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_mul_f32_e32 v7, 0x4f7ffffe, v7
-; CGP-NEXT: v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT: v_mul_lo_u32 v8, v8, v3
-; CGP-NEXT: v_mul_hi_u32 v8, v3, v8
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v8
-; CGP-NEXT: v_mul_hi_u32 v3, v3, v0
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT: v_mul_lo_u32 v8, v8, v7
-; CGP-NEXT: v_mul_u32_u24_e32 v4, v3, v4
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v3
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT: v_cvt_f32_u32_e32 v3, v5
+; CGP-NEXT: v_and_b32_e32 v7, 0xffffff, v0
+; CGP-NEXT: v_rcp_f32_e32 v1, v1
+; CGP-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; CGP-NEXT: v_cvt_u32_f32_e32 v6, v1
+; CGP-NEXT: v_rcp_f32_e32 v1, v3
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v4
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v6
+; CGP-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1
+; CGP-NEXT: v_cvt_u32_f32_e32 v8, v0
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, 0, v5
+; CGP-NEXT: v_mul_lo_u32 v9, v0, v8
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v6, v1
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v3, 0
+; CGP-NEXT: v_and_b32_e32 v6, 0xffffff, v2
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v9, 0
+; CGP-NEXT: v_mul_lo_u32 v0, v1, v4
+; CGP-NEXT: v_add_i32_e32 v2, vcc, 1, v1
+; CGP-NEXT: v_add_i32_e64 v3, s[4:5], v8, v3
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v7, v0
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
+; CGP-NEXT: v_cndmask_b32_e32 v7, v1, v2, vcc
+; CGP-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v6, v3, 0
+; CGP-NEXT: v_sub_i32_e64 v9, s[4:5], v0, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
+; CGP-NEXT: v_mul_lo_u32 v3, v2, v5
+; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v7
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v7, v1, vcc
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v6, v3
+; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v5
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v5
; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; CGP-NEXT: v_mul_hi_u32 v4, v7, v8
-; CGP-NEXT: v_sub_i32_e64 v8, s[4:5], v0, v1
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v7, v4
-; CGP-NEXT: v_mul_hi_u32 v4, v4, v2
-; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v3
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
-; CGP-NEXT: v_mul_u32_u24_e32 v1, v4, v6
-; CGP-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v2, v1
-; CGP-NEXT: v_add_i32_e32 v2, vcc, 1, v4
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
-; CGP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v1, v5
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v2
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
-; CGP-NEXT: v_mov_b32_e32 v1, 0
-; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v5
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; CGP-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; CGP-NEXT: s_setpc_b64 s[30:31]
%num.mask = and <2 x i64> %num, <i64 16777215, i64 16777215>
%den.mask = and <2 x i64> %den, <i64 16777215, i64 16777215>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
index 19101498e5f1a..65fecc829cbb5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
@@ -645,16 +645,30 @@ define amdgpu_ps i32 @s_sext_inreg_v2i16_11(<2 x i16> inreg %value) {
;
; GFX9-LABEL: s_sext_inreg_v2i16_11:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, 11, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, 11, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0xb000b
+; GFX9-NEXT: s_lshl_b32 s1, s1, 11
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT: s_sext_i32_i16 s1, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s2, 0xb000b
+; GFX9-NEXT: s_ashr_i32 s1, s1, s2
+; GFX9-NEXT: s_ashr_i32 s0, s0, 11
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_sext_inreg_v2i16_11:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v0, 11, s0 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v0, 11, v0 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 0xb000b
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 11
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX10PLUS-NEXT: s_sext_i32_i16 s1, 0xb000b
+; GFX10PLUS-NEXT: s_sext_i32_i16 s2, s0
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s2, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 11
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX10PLUS-NEXT: ; return to shader part epilog
%shl = shl <2 x i16> %value, <i16 11, i16 11>
%ashr = ashr <2 x i16> %shl, <i16 11, i16 11>
@@ -767,22 +781,48 @@ define amdgpu_ps <2 x i32> @s_sext_inreg_v4i16_14(<4 x i16> inreg %value) {
;
; GFX9-LABEL: s_sext_inreg_v4i16_14:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, 14, s1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, 14, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, 14, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, 14, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_readfirstlane_b32 s0, v1
-; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0xe000e
+; GFX9-NEXT: s_lshl_b32 s2, s2, 14
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_sext_i32_i16 s2, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s3, 0xe000e
+; GFX9-NEXT: s_ashr_i32 s2, s2, s3
+; GFX9-NEXT: s_ashr_i32 s0, s0, 14
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX9-NEXT: s_lshr_b32 s2, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0xe000e
+; GFX9-NEXT: s_lshl_b32 s2, s2, 14
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT: s_sext_i32_i16 s2, s1
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_ashr_i32 s2, s2, s3
+; GFX9-NEXT: s_ashr_i32 s1, s1, 14
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_sext_inreg_v4i16_14:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v0, 14, s0 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v1, 14, s1 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v0, 14, v0 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v1, 14, v1 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s1, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 0xe000e
+; GFX10PLUS-NEXT: s_lshl_b32 s2, s2, 14
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 0xe000e
+; GFX10PLUS-NEXT: s_lshl_b32 s4, s4, 14
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX10PLUS-NEXT: s_sext_i32_i16 s2, 0xe000e
+; GFX10PLUS-NEXT: s_sext_i32_i16 s3, s0
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s4, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s3, s3, s2
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 14
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s4, s2
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, 14
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX10PLUS-NEXT: ; return to shader part epilog
%shl = shl <4 x i16> %value, <i16 14, i16 14, i16 14, i16 14>
%ashr = ashr <4 x i16> %shl, <i16 14, i16 14, i16 14, i16 14>
@@ -959,34 +999,84 @@ define amdgpu_ps <4 x i32> @s_sext_inreg_v8i16_5(<8 x i16> inreg %value) {
;
; GFX9-LABEL: s_sext_inreg_v8i16_5:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, 5, s3 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, 5, s2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 5, s1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_lshlrev_b16 v3, 5, s0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v0, 5, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, 5, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v2, 5, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_ashrrev_i16 v3, 5, v3 op_sel_hi:[0,1]
-; GFX9-NEXT: v_readfirstlane_b32 s0, v3
-; GFX9-NEXT: v_readfirstlane_b32 s1, v2
-; GFX9-NEXT: v_readfirstlane_b32 s2, v1
-; GFX9-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x50005
+; GFX9-NEXT: s_lshl_b32 s4, s4, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_sext_i32_i16 s4, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s5, 0x50005
+; GFX9-NEXT: s_ashr_i32 s4, s4, s5
+; GFX9-NEXT: s_ashr_i32 s0, s0, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x50005
+; GFX9-NEXT: s_lshl_b32 s4, s4, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_sext_i32_i16 s4, s1
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_ashr_i32 s4, s4, s5
+; GFX9-NEXT: s_ashr_i32 s1, s1, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s4, s1
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s2, 0x50005
+; GFX9-NEXT: s_lshl_b32 s4, s4, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT: s_sext_i32_i16 s4, s2
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: s_ashr_i32 s4, s4, s5
+; GFX9-NEXT: s_ashr_i32 s2, s2, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshl_b32 s3, s3, 0x50005
+; GFX9-NEXT: s_lshl_b32 s4, s4, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT: s_sext_i32_i16 s4, s3
+; GFX9-NEXT: s_ashr_i32 s3, s3, 16
+; GFX9-NEXT: s_ashr_i32 s4, s4, s5
+; GFX9-NEXT: s_ashr_i32 s3, s3, 5
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s4, s3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_sext_inreg_v8i16_5:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v0, 5, s0 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v1, 5, s1 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v2, 5, s2 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v3, 5, s3 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v0, 5, v0 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v1, 5, v1 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v2, 5, v2 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_pk_ashrrev_i16 v3, 5, v3 op_sel_hi:[0,1]
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v2
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s6, s1, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 0x50005
+; GFX10PLUS-NEXT: s_lshl_b32 s4, s4, 5
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 0x50005
+; GFX10PLUS-NEXT: s_lshl_b32 s6, s6, 5
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s6
+; GFX10PLUS-NEXT: s_lshr_b32 s6, s2, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s4, s0
+; GFX10PLUS-NEXT: s_sext_i32_i16 s5, 0x50005
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s2, s2, 0x50005
+; GFX10PLUS-NEXT: s_lshl_b32 s6, s6, 5
+; GFX10PLUS-NEXT: s_ashr_i32 s4, s4, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 5
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s2, s2, s6
+; GFX10PLUS-NEXT: s_lshr_b32 s6, s3, 16
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s4, s0
+; GFX10PLUS-NEXT: s_sext_i32_i16 s4, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s3, s3, 0x50005
+; GFX10PLUS-NEXT: s_lshl_b32 s6, s6, 5
+; GFX10PLUS-NEXT: s_ashr_i32 s4, s4, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s1, 5
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s3, s3, s6
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s4, s1
+; GFX10PLUS-NEXT: s_sext_i32_i16 s4, s2
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s2, 16
+; GFX10PLUS-NEXT: s_sext_i32_i16 s6, s3
+; GFX10PLUS-NEXT: s_ashr_i32 s3, s3, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s4, s4, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s2, 5
+; GFX10PLUS-NEXT: s_ashr_i32 s5, s6, s5
+; GFX10PLUS-NEXT: s_ashr_i32 s3, s3, 5
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s3, s5, s3
; GFX10PLUS-NEXT: ; return to shader part epilog
%shl = shl <8 x i16> %value, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
%ashr = ashr <8 x i16> %shl, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
index 7a8eecb5bae39..edd665c99f5a4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
@@ -3,7 +3,7 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; Test optimization to reduce shifts to narrower sizes.
@@ -181,20 +181,16 @@ define i64 @v_shl_i64_zext_i32_overflow(i32 %x) {
define amdgpu_ps i64 @s_shl_i64_sext_i32_overflow(i32 inreg %x) {
; GCN-LABEL: s_shl_i64_sext_i32_overflow:
; GCN: ; %bb.0:
-; GCN-NEXT: s_mov_b32 s2, s0
; GCN-NEXT: s_bitset0_b32 s0, 31
-; GCN-NEXT: s_mov_b32 s1, 0
+; GCN-NEXT: s_ashr_i32 s1, s0, 31
; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN-NEXT: s_bfe_u32 s1, s2, 0x1001e
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_shl_i64_sext_i32_overflow:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_mov_b32 s2, s0
-; GFX10PLUS-NEXT: s_mov_b32 s1, 0
; GFX10PLUS-NEXT: s_bitset0_b32 s0, 31
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s0, 31
; GFX10PLUS-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX10PLUS-NEXT: s_bfe_u32 s1, s2, 0x1001e
; GFX10PLUS-NEXT: ; return to shader part epilog
%and = and i32 %x, 2147483647
%ext = sext i32 %and to i64
@@ -206,51 +202,34 @@ define i64 @v_shl_i64_sext_i32_overflow(i32 %x) {
; GFX7-LABEL: v_shl_i64_sext_i32_overflow:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, 0
+; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX7-NEXT: v_bfe_u32 v1, v2, 30, 1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_shl_i64_sext_i32_overflow:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_and_b32_e32 v0, 0x7fffffff, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX8-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX8-NEXT: v_bfe_u32 v1, v2, 30, 1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_shl_i64_sext_i32_overflow:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffffff, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX9-NEXT: v_bfe_u32 v1, v2, 30, 1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_shl_i64_sext_i32_overflow:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0x7fffffff, v2
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX10-NEXT: v_bfe_u32 v1, v2, 30, 1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_shl_i64_sext_i32_overflow:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x7fffffff, v2
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX11-NEXT: v_bfe_u32 v1, v2, 30, 1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10PLUS-LABEL: v_shl_i64_sext_i32_overflow:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10PLUS-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX10PLUS-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%and = and i32 %x, 2147483647
%ext = sext i32 %and to i64
%shl = shl i64 %ext, 2
@@ -581,18 +560,21 @@ define amdgpu_ps i32 @s_shl_i32_zext_i16(i16 inreg %x) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s0, 0x3fff
; GFX8-NEXT: s_lshl_b32 s0, s0, 2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_shl_i32_zext_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s0, 0x3fff
; GFX9-NEXT: s_lshl_b32 s0, s0, 2
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_shl_i32_zext_i16:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0x3fff
; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 2
+; GFX10PLUS-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10PLUS-NEXT: ; return to shader part epilog
%and = and i16 %x, 16383
%ext = zext i16 %and to i32
@@ -663,10 +645,12 @@ define amdgpu_ps <2 x i32> @s_shl_v2i32_zext_v2i16(<2 x i16> inreg %x) {
;
; GFX8-LABEL: s_shl_v2i32_zext_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s0, 0x3fff3fff
-; GFX8-NEXT: s_and_b32 s0, s1, 0x3fff
+; GFX8-NEXT: s_and_b32 s0, s0, 0x3fff3fff
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
; GFX8-NEXT: s_lshl_b32 s0, s0, 2
-; GFX8-NEXT: s_lshr_b32 s1, s1, 14
+; GFX8-NEXT: s_lshl_b32 s1, s1, 2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_shl_v2i32_zext_v2i16:
@@ -677,13 +661,32 @@ define amdgpu_ps <2 x i32> @s_shl_v2i32_zext_v2i16(<2 x i16> inreg %x) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 14
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_shl_v2i32_zext_v2i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s1, s0, 0x3fff3fff
-; GFX10PLUS-NEXT: s_and_b32 s0, s1, 0x3fff
-; GFX10PLUS-NEXT: s_lshr_b32 s1, s1, 14
-; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 2
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_shl_v2i32_zext_v2i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s1, s0, 0x3fff3fff
+; GFX10-NEXT: s_and_b32 s0, s1, 0x3fff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 14
+; GFX10-NEXT: s_lshl_b32 s0, s0, 2
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_shl_v2i32_zext_v2i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x3fff3fff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x20002
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_shl_v2i32_zext_v2i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s0, 0x3fff3fff
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s1, 0x3fff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 14
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%and = and <2 x i16> %x, <i16 16383, i16 16383>
%ext = zext <2 x i16> %and to <2 x i32>
%shl = shl <2 x i32> %ext, <i32 2, i32 2>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
index 4fb716f5574bf..b1e706583d445 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
@@ -790,15 +790,20 @@ define amdgpu_ps i32 @s_shl_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amoun
;
; GFX9-LABEL: s_shl_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, s1, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s1, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_shl_v2i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v0, s1, s0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, s1
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s2, s3
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = shl <2 x i16> %value, %amount
%cast = bitcast <2 x i16> %result to i32
@@ -974,20 +979,30 @@ define amdgpu_ps <2 x i32> @s_shl_v4i16(<4 x i16> inreg %value, <4 x i16> inreg
;
; GFX9-LABEL: s_shl_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, s3, v0
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, s2, v1
-; GFX9-NEXT: v_readfirstlane_b32 s0, v1
-; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s2, s4, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s2, s1, 16
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, s3
+; GFX9-NEXT: s_lshl_b32 s2, s2, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_shl_v4i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v0, s2, s0
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v1, s3, s1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s2, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, s2
+; GFX10PLUS-NEXT: s_lshl_b32 s2, s4, s5
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s1, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s3, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, s3
+; GFX10PLUS-NEXT: s_lshl_b32 s3, s4, s5
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s3
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = shl <4 x i16> %value, %amount
%cast = bitcast <4 x i16> %result to <2 x i32>
@@ -1157,30 +1172,50 @@ define amdgpu_ps <4 x i32> @s_shl_v8i16(<8 x i16> inreg %value, <8 x i16> inreg
;
; GFX9-LABEL: s_shl_v8i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, s0
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, s7, v0
-; GFX9-NEXT: v_pk_lshlrev_b16 v1, s6, v1
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, s5, v2
-; GFX9-NEXT: v_pk_lshlrev_b16 v3, s4, v3
-; GFX9-NEXT: v_readfirstlane_b32 s0, v3
-; GFX9-NEXT: v_readfirstlane_b32 s1, v2
-; GFX9-NEXT: v_readfirstlane_b32 s2, v1
-; GFX9-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-NEXT: s_lshr_b32 s8, s0, 16
+; GFX9-NEXT: s_lshr_b32 s9, s4, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s4
+; GFX9-NEXT: s_lshl_b32 s4, s8, s9
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_lshr_b32 s8, s5, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, s5
+; GFX9-NEXT: s_lshl_b32 s4, s4, s8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_lshr_b32 s5, s6, 16
+; GFX9-NEXT: s_lshl_b32 s2, s2, s6
+; GFX9-NEXT: s_lshl_b32 s4, s4, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshr_b32 s5, s7, 16
+; GFX9-NEXT: s_lshl_b32 s3, s3, s7
+; GFX9-NEXT: s_lshl_b32 s4, s4, s5
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s4
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_shl_v8i16:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v0, s4, s0
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v1, s5, s1
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v2, s6, s2
-; GFX10PLUS-NEXT: v_pk_lshlrev_b16 v3, s7, s3
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v1
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v2
-; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10PLUS-NEXT: s_lshr_b32 s8, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s9, s4, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, s4
+; GFX10PLUS-NEXT: s_lshl_b32 s4, s8, s9
+; GFX10PLUS-NEXT: s_lshr_b32 s8, s1, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s9, s5, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, s5
+; GFX10PLUS-NEXT: s_lshl_b32 s5, s8, s9
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s2, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s6, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s2, s2, s6
+; GFX10PLUS-NEXT: s_lshl_b32 s4, s4, s5
+; GFX10PLUS-NEXT: s_lshr_b32 s5, s3, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s6, s7, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s3, s3, s7
+; GFX10PLUS-NEXT: s_lshl_b32 s5, s5, s6
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s3, s3, s5
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = shl <8 x i16> %value, %amount
%cast = bitcast <8 x i16> %result to <4 x i32>
@@ -1525,58 +1560,55 @@ define i65 @v_shl_i65(i65 %value, i65 %amount) {
; GFX6-LABEL: v_shl_i65:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[2:3], v3
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, 64, v3
-; GFX6-NEXT: v_lshr_b64 v[5:6], v[0:1], v5
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
-; GFX6-NEXT: v_subrev_i32_e32 v4, vcc, 64, v3
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[0:1], v4
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, 64, v3
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[0:1], v4
+; GFX6-NEXT: v_lshl_b64 v[5:6], v[2:3], v3
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0xffffffc0, v3
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[0:1], v3
+; GFX6-NEXT: v_or_b32_e32 v9, v4, v5
+; GFX6-NEXT: v_lshl_b64 v[4:5], v[0:1], v8
; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], v3
-; GFX6-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; GFX6-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v7, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_shl_i65:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], v3, v[2:3]
-; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 64, v3
-; GFX8-NEXT: v_lshrrev_b64 v[5:6], v5, v[0:1]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_or_b32_e32 v6, v4, v5
-; GFX8-NEXT: v_subrev_u32_e32 v4, vcc, 64, v3
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], v4, v[0:1]
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, 64, v3
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v4, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[5:6], v3, v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0xffffffc0, v3
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], v3, v[0:1]
+; GFX8-NEXT: v_or_b32_e32 v9, v4, v5
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], v8, v[0:1]
; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v3, v[0:1]
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v7, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_shl_i65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], v3, v[2:3]
-; GFX9-NEXT: v_sub_u32_e32 v5, 64, v3
-; GFX9-NEXT: v_lshrrev_b64 v[5:6], v5, v[0:1]
+; GFX9-NEXT: v_sub_u32_e32 v4, 64, v3
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], v4, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[5:6], v3, v[2:3]
+; GFX9-NEXT: v_add_u32_e32 v8, 0xffffffc0, v3
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], v3, v[0:1]
+; GFX9-NEXT: v_or_b32_e32 v9, v4, v5
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v8, v[0:1]
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX9-NEXT: v_or_b32_e32 v6, v4, v5
-; GFX9-NEXT: v_subrev_u32_e32 v4, 64, v3
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], v4, v[0:1]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v3, v[0:1]
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_shl_i65:
@@ -1584,18 +1616,17 @@ define i65 @v_shl_i65(i65 %value, i65 %amount) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_sub_nc_u32_e32 v6, 64, v3
; GFX10-NEXT: v_lshlrev_b64 v[4:5], v3, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v8, 0xffffffc0, v3
; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v3
; GFX10-NEXT: v_lshrrev_b64 v[5:6], v6, v[0:1]
-; GFX10-NEXT: v_subrev_nc_u32_e32 v6, 64, v3
-; GFX10-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
-; GFX10-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], v3, v[0:1]
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, v2, s4
-; GFX10-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1]
+; GFX10-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v8, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v7, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_shl_i65:
@@ -1603,38 +1634,65 @@ define i65 @v_shl_i65(i65 %value, i65 %amount) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_sub_nc_u32_e32 v6, 64, v3
; GFX11-NEXT: v_lshlrev_b64 v[4:5], v3, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v8, 0xffffffc0, v3
; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
; GFX11-NEXT: v_lshrrev_b64 v[5:6], v6, v[0:1]
-; GFX11-NEXT: v_subrev_nc_u32_e32 v6, 64, v3
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v3, v[0:1]
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v1, 0, v1
-; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v2, s0
-; GFX11-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], v3, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1]
+; GFX11-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v8, v1 :: v_dual_cndmask_b32 v1, 0, v7
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = shl i65 %value, %amount
ret i65 %result
}
define i65 @v_shl_i65_33(i65 %value) {
-; GCN-LABEL: v_shl_i65_33:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 31, v0
-; GCN-NEXT: v_lshlrev_b32_e32 v1, 1, v0
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX6-LABEL: v_shl_i65_33:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 1, v0
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[0:1], 31
+; GFX6-NEXT: v_mov_b32_e32 v0, 0
+; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_shl_i65_33:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_lshlrev_b32_e32 v1, 1, v0
-; GFX10PLUS-NEXT: v_lshrrev_b32_e32 v2, 31, v0
-; GFX10PLUS-NEXT: v_mov_b32_e32 v0, 0
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: v_shl_i65_33:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 1, v0
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], 31, v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_shl_i65_33:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 1, v0
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], 31, v[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_shl_i65_33:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 1, v0
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], 31, v[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_shl_i65_33:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 1, v0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 31, v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, v4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = shl i65 %value, 33
ret i65 %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll
index 204dbff605229..3e78f48e37a50 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=verde < %s | FileCheck -check-prefix=SI %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
define amdgpu_ps i64 @s_mul_u64_u32(i32 inreg %a, i32 inreg %b) {
; SI-LABEL: s_mul_u64_u32:
@@ -173,17 +173,18 @@ define i64 @v_mul_i64_i32(i32 %a, i32 %b) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mad_co_i64_i32 v[0:1], null, v0, v1, 0
+; GFX12-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_mad_co_i64_i32 v[0:1], null, v2, v3, 0
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_mul_i64_i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_ashrrev_i32 v1, 31, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_mad_nc_i64_i32 v[0:1], v2, v3, 0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ext_a = sext i32 %a to i64
%ext_b = sext i32 %b to i64
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
index 5e5bddc4b7956..27f58ff9f0438 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 < %s | FileCheck -check-prefixes=CHECK,CGP %s
; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
@@ -282,16 +282,16 @@ define <2 x i32> @v_srem_v2i32_pow2k_denom(<2 x i32> %num) {
; GISEL-LABEL: v_srem_v2i32_pow2k_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v2, 20, v2
-; GISEL-NEXT: v_lshrrev_b32_e32 v3, 20, v3
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v1, v2
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v0, v3
-; GISEL-NEXT: v_and_b32_e32 v2, 0xfffff000, v2
-; GISEL-NEXT: v_and_b32_e32 v3, 0xfffff000, v3
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
+; GISEL-NEXT: v_ashrrev_i32_e32 v2, 31, v0
+; GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v1
+; GISEL-NEXT: v_and_b32_e32 v2, 0xfff, v2
+; GISEL-NEXT: v_and_b32_e32 v3, 0xfff, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v3
+; GISEL-NEXT: v_and_b32_e32 v0, 0xfff, v0
+; GISEL-NEXT: v_and_b32_e32 v1, 0xfff, v1
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i32_pow2k_denom:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index 1696c688e2585..192fc0ab9dc10 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -8,146 +8,175 @@ define i64 @v_srem_i64(i64 %num, i64 %den) {
; CHECK-LABEL: v_srem_i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_or_b32_e32 v4, v1, v3
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: v_mov_b32_e32 v5, v1
+; CHECK-NEXT: v_mov_b32_e32 v4, v0
+; CHECK-NEXT: v_or_b32_e32 v1, v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CHECK-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB0_2
-; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
-; CHECK-NEXT: v_xor_b32_e32 v7, v3, v4
-; CHECK-NEXT: v_xor_b32_e32 v8, v2, v4
-; CHECK-NEXT: v_cvt_f32_u32_e32 v2, v8
-; CHECK-NEXT: v_cvt_f32_u32_e32 v3, v7
-; CHECK-NEXT: v_sub_i32_e32 v11, vcc, 0, v8
-; CHECK-NEXT: v_subb_u32_e32 v12, vcc, 0, v7, vcc
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
-; CHECK-NEXT: v_rcp_f32_e32 v2, v2
+; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_3
+; CHECK-NEXT: ; %bb.1: ; %Flow
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_4
+; CHECK-NEXT: .LBB0_2: ; %.split
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; CHECK-NEXT: .LBB0_3:
+; CHECK-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v2, v1
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v3, v1, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1
+; CHECK-NEXT: v_xor_b32_e32 v1, v2, v1
+; CHECK-NEXT: v_cvt_f32_u32_e32 v2, v0
+; CHECK-NEXT: v_cvt_f32_u32_e32 v3, v1
+; CHECK-NEXT: v_sub_i32_e32 v12, vcc, 0, v0
+; CHECK-NEXT: v_subb_u32_e32 v13, vcc, 0, v1, vcc
+; CHECK-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v2, v2
; CHECK-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
; CHECK-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
; CHECK-NEXT: v_trunc_f32_e32 v3, v3
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
-; CHECK-NEXT: v_cvt_u32_f32_e32 v9, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v10, v2
-; CHECK-NEXT: v_mul_lo_u32 v4, v11, v9
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v10, 0
-; CHECK-NEXT: v_mul_lo_u32 v5, v12, v10
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v13, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v13, 0
-; CHECK-NEXT: v_mul_hi_u32 v14, v10, v2
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v14, vcc, v14, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v13, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v14, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v2
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, v9, v3, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v10, 0
-; CHECK-NEXT: v_mul_lo_u32 v4, v11, v9
-; CHECK-NEXT: v_mul_lo_u32 v5, v12, v10
-; CHECK-NEXT: v_mul_hi_u32 v12, v10, v2
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v11, 0
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v12, vcc, v12, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v11, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v12, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v9, v3, vcc
-; CHECK-NEXT: v_ashrrev_i32_e32 v5, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_xor_b32_e32 v6, v0, v5
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v1, v5, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
-; CHECK-NEXT: v_mul_hi_u32 v9, v6, v2
-; CHECK-NEXT: v_xor_b32_e32 v10, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v0
-; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v4, 0
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v0
-; CHECK-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; CHECK-NEXT: v_mul_lo_u32 v3, v8, v0
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v2, 0
-; CHECK-NEXT: v_mul_lo_u32 v2, v7, v2
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v10, v1
+; CHECK-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v11, v2
+; CHECK-NEXT: v_cvt_u32_f32_e32 v10, v3
+; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
+; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v12, v10, v[3:4]
+; CHECK-NEXT: v_mul_hi_u32 v3, v11, v2
+; CHECK-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v13, v11, v[6:7]
+; CHECK-NEXT: v_mul_lo_u32 v6, v10, v2
+; CHECK-NEXT: v_mul_hi_u32 v2, v10, v2
+; CHECK-NEXT: v_mul_lo_u32 v7, v11, v8
+; CHECK-NEXT: v_mul_hi_u32 v9, v11, v8
+; CHECK-NEXT: v_mul_lo_u32 v14, v10, v8
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v9
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v14
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_mul_hi_u32 v7, v10, v8
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
+; CHECK-NEXT: v_add_i32_e32 v11, vcc, v11, v2
+; CHECK-NEXT: v_addc_u32_e32 v10, vcc, v10, v3, vcc
+; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
+; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v12, v10, v[3:4]
+; CHECK-NEXT: v_ashrrev_i32_e32 v12, 31, v5
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v12
+; CHECK-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v13, v11, v[6:7]
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v5, v12, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v11, v2
+; CHECK-NEXT: v_mul_lo_u32 v6, v11, v8
+; CHECK-NEXT: v_xor_b32_e32 v9, v3, v12
+; CHECK-NEXT: v_mul_lo_u32 v3, v10, v2
+; CHECK-NEXT: v_mul_hi_u32 v2, v10, v2
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v11, v8
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT: v_mul_lo_u32 v6, v10, v8
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v10, v8
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v11, v2
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v9, v2
+; CHECK-NEXT: v_mul_lo_u32 v6, v9, v3
+; CHECK-NEXT: v_xor_b32_e32 v8, v4, v12
+; CHECK-NEXT: v_mul_lo_u32 v4, v8, v2
+; CHECK-NEXT: v_mul_hi_u32 v2, v8, v2
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v9, v3
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT: v_mul_lo_u32 v6, v8, v3
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v2, v4
+; CHECK-NEXT: v_mul_hi_u32 v6, v8, v3
+; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v10, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v4
+; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v6, v[3:4]
+; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v9, v2
+; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v10, v[4:5]
+; CHECK-NEXT: v_subb_u32_e64 v3, s[4:5], v8, v6, vcc
+; CHECK-NEXT: v_sub_i32_e64 v4, s[4:5], v8, v6
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v0
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v3, v1
+; CHECK-NEXT: v_subb_u32_e32 v4, vcc, v4, v1, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[4:5]
+; CHECK-NEXT: v_sub_i32_e32 v6, vcc, v2, v0
+; CHECK-NEXT: v_subbrev_u32_e64 v7, s[4:5], 0, v4, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v0
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v1
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v4, v1, vcc
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v6, v0
-; CHECK-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v7, vcc
-; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v8
-; CHECK-NEXT: v_subbrev_u32_e64 v4, s[6:7], 0, v2, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[6:7], v4, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[6:7]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v7
-; CHECK-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v7, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[6:7]
-; CHECK-NEXT: v_sub_i32_e64 v9, s[4:5], v3, v8
-; CHECK-NEXT: v_subbrev_u32_e64 v2, s[4:5], 0, v2, s[4:5]
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v10, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v1, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v1, v7
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v2, v3, v9, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; CHECK-NEXT: v_xor_b32_e32 v0, v0, v5
-; CHECK-NEXT: v_xor_b32_e32 v1, v1, v5
-; CHECK-NEXT: v_sub_i32_e32 v4, vcc, v0, v5
-; CHECK-NEXT: v_subb_u32_e32 v5, vcc, v1, v5, vcc
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: .LBB0_2: ; %Flow
-; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; CHECK-NEXT: s_cbranch_execz .LBB0_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v3, vcc, 0, v2
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_lo_u32 v3, v3, v1
-; CHECK-NEXT: v_mul_hi_u32 v3, v1, v3
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[4:5]
+; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v12
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v12
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v12
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v12, vcc
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT: s_cbranch_execz .LBB0_2
+; CHECK-NEXT: .LBB0_4:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, v2
+; CHECK-NEXT: v_sub_i32_e32 v1, vcc, 0, v2
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v1, v0
; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_mul_lo_u32 v1, v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mul_lo_u32 v0, v0, v2
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v4, v0
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v0, v2
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v0, v2
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
-; CHECK-NEXT: .LBB0_4: ; %.split
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v0, v4
-; CHECK-NEXT: v_mov_b32_e32 v1, v5
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = srem i64 %num, %den
ret i64 %result
@@ -330,337 +359,412 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v5
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v8, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v8
-; GISEL-NEXT: v_xor_b32_e32 v8, v4, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v5
-; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v8
-; GISEL-NEXT: v_subb_u32_e32 v16, vcc, 0, v5, vcc
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v5, v8, vcc
+; GISEL-NEXT: v_xor_b32_e32 v5, v4, v8
+; GISEL-NEXT: v_xor_b32_e32 v8, v9, v8
+; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v5
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v8
+; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 0, v5
+; GISEL-NEXT: v_subb_u32_e32 v17, vcc, 0, v8, vcc
; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v9
-; GISEL-NEXT: v_rcp_f32_e32 v4, v4
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v4
; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
; GISEL-NEXT: v_mul_f32_e32 v9, 0x2f800000, v4
; GISEL-NEXT: v_trunc_f32_e32 v9, v9
; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v9
; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v9
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v4, 0
-; GISEL-NEXT: v_mul_lo_u32 v11, v15, v14
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v4
-; GISEL-NEXT: v_mul_hi_u32 v18, v4, v9
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v10, v12
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v4, v17, 0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v14, v9, 0
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v18, v10
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v14, v17, 0
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v18, v12
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, v11, v13, vcc
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v9
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v14, v10, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v4, 0
-; GISEL-NEXT: v_mul_lo_u32 v11, v15, v14
-; GISEL-NEXT: v_mul_lo_u32 v12, v16, v4
-; GISEL-NEXT: v_mul_hi_u32 v16, v4, v9
+; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v16, v4, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v15, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v10, v4, v9
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v4, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v12, v15, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v9
+; GISEL-NEXT: v_mul_lo_u32 v11, v4, v13
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v10, v12
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v4, v15, 0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v14, v9, 0
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v16, v10
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v14, v15, 0
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v16, v12
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, v11, v13, vcc
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v4, v9
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, v14, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v12, v4, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_mul_hi_u32 v12, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v4, v9
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, v15, v10, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v16, v18, 0
; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_xor_b32_e32 v12, v0, v4
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v1, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
-; GISEL-NEXT: v_mul_hi_u32 v13, v12, v9
-; GISEL-NEXT: v_xor_b32_e32 v14, v10, v4
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v14, v9, 0
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v0
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v11, 0
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v15, v10, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v10, v8, v0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v9, 0
-; GISEL-NEXT: v_mul_lo_u32 v9, v5, v9
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v10
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v9
-; GISEL-NEXT: v_sub_i32_e32 v9, vcc, v14, v1
-; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v12, v0
-; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], v9, v5, vcc
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v10, v8
-; GISEL-NEXT: v_subbrev_u32_e64 v11, s[6:7], 0, v0, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v11, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v9, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v11, v5
-; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], v0, v5, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v13, s[4:5], v9, v8
-; GISEL-NEXT: v_subbrev_u32_e64 v0, s[4:5], 0, v0, s[4:5]
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v0, v11, v0, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v14, v1, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v11, 31, v7
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v6, v11
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v7, v11, vcc
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v11
-; GISEL-NEXT: v_xor_b32_e32 v11, v12, v11
-; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v11
-; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v6
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v10, v8
-; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v12
-; GISEL-NEXT: v_rcp_f32_e32 v7, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v14, v8, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v1, v0, vcc
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v7
-; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v1, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
-; GISEL-NEXT: v_cvt_u32_f32_e32 v12, v0
-; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v1
-; GISEL-NEXT: v_sub_i32_e64 v15, s[6:7], 0, v11
-; GISEL-NEXT: v_subb_u32_e64 v16, s[6:7], 0, v6, s[6:7]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v15, v12, 0
-; GISEL-NEXT: v_mul_lo_u32 v7, v15, v14
-; GISEL-NEXT: v_mul_lo_u32 v8, v16, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v9, v9, v13, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v17, v12, v0
-; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v1, v7
-; GISEL-NEXT: v_add_i32_e64 v13, s[4:5], v1, v8
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v13, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v0, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v9, v9, v4
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v17, v7
-; GISEL-NEXT: v_addc_u32_e32 v17, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v14, v13, 0
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v15, v[10:11]
+; GISEL-NEXT: v_mul_hi_u32 v10, v18, v9
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v18, v[11:12]
+; GISEL-NEXT: v_xor_b32_e32 v14, v0, v4
+; GISEL-NEXT: v_mul_lo_u32 v0, v15, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v9
+; GISEL-NEXT: v_mul_lo_u32 v11, v18, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v17, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v8, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v12, v0
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, v14, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v10, 0
-; GISEL-NEXT: v_mul_lo_u32 v7, v15, v12
-; GISEL-NEXT: v_mul_lo_u32 v8, v16, v10
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
-; GISEL-NEXT: v_mul_hi_u32 v14, v10, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v7
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v1, v8
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v13, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v0, 0
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v7
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v13, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v18, v13
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
+; GISEL-NEXT: v_mul_lo_u32 v11, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v18, v0
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v15, v9, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v14, v0
+; GISEL-NEXT: v_mul_lo_u32 v11, v14, v9
+; GISEL-NEXT: v_xor_b32_e32 v13, v1, v4
+; GISEL-NEXT: v_mul_lo_u32 v1, v13, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v13, v0
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v10, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v14, v9
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
+; GISEL-NEXT: v_mul_lo_u32 v11, v13, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v9, v13, v9
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v15, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v9, v10
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v5, v11, v[1:2]
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v8, v15, v[9:10]
+; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v14, v0
+; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], v13, v11, vcc
+; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v13, v11
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v5
+; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v0, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v12, v8
+; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v10, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v1, v9, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e64 v14, s[4:5], 0, v0, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v7
+; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v1
+; GISEL-NEXT: v_addc_u32_e64 v7, s[4:5], v7, v1, s[4:5]
+; GISEL-NEXT: v_xor_b32_e32 v6, v6, v1
+; GISEL-NEXT: v_xor_b32_e32 v7, v7, v1
+; GISEL-NEXT: v_cvt_f32_u32_e32 v1, v6
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v7
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v14, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v13, v5
+; GISEL-NEXT: v_mac_f32_e32 v1, 0x4f800000, v9
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v14, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v15, v15, v16, s[4:5]
+; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v0, v8, vcc
+; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
+; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GISEL-NEXT: v_trunc_f32_e32 v8, v1
+; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v8
+; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v0
+; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 0, v6
+; GISEL-NEXT: v_cvt_u32_f32_e32 v20, v8
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v17, 0
+; GISEL-NEXT: v_subb_u32_e32 v19, vcc, 0, v7, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v18, v20, v[1:2]
+; GISEL-NEXT: v_mul_hi_u32 v21, v17, v0
+; GISEL-NEXT: v_mul_lo_u32 v22, v20, v0
+; GISEL-NEXT: v_mul_hi_u32 v23, v20, v0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v13, v5
+; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v16, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v15
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v0, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v19, v17, v[8:9]
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v14, v5, vcc
+; GISEL-NEXT: v_mul_lo_u32 v1, v17, v0
+; GISEL-NEXT: v_mul_hi_u32 v9, v17, v0
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v21, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v22
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
+; GISEL-NEXT: v_mul_lo_u32 v8, v20, v0
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v23, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v14, v9
+; GISEL-NEXT: v_mul_hi_u32 v0, v20, v0
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v8, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v17, v1
+; GISEL-NEXT: v_addc_u32_e32 v15, vcc, v20, v0, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v14, 0
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v13, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v12, v5, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v18, v15, v[1:2]
+; GISEL-NEXT: v_xor_b32_e32 v1, v10, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v12, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v12
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v19, v14, v[8:9]
+; GISEL-NEXT: v_mul_hi_u32 v8, v14, v0
+; GISEL-NEXT: v_xor_b32_e32 v11, v2, v12
+; GISEL-NEXT: v_mul_lo_u32 v2, v15, v0
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, v10
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v12, vcc
+; GISEL-NEXT: v_mul_hi_u32 v0, v15, v0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v14, v10
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
+; GISEL-NEXT: v_mul_lo_u32 v9, v15, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v15, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v8, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v0
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v12, v1, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v10, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v10
-; GISEL-NEXT: v_xor_b32_e32 v12, v0, v10
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
-; GISEL-NEXT: v_mul_hi_u32 v13, v12, v7
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v3, v10, vcc
-; GISEL-NEXT: v_xor_b32_e32 v14, v2, v10
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v14, v7, 0
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v13, v0
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v7, v2
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v13, v3, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v2, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v8, v11, v0
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v7, 0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v9, v4
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v15, v2, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v11, v0
+; GISEL-NEXT: v_mul_lo_u32 v9, v11, v2
+; GISEL-NEXT: v_xor_b32_e32 v10, v3, v12
+; GISEL-NEXT: v_mul_lo_u32 v3, v10, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v10, v0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v8, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v8, v11, v2
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
+; GISEL-NEXT: v_mul_lo_u32 v9, v10, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
+; GISEL-NEXT: v_mul_hi_u32 v9, v10, v2
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v13, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v0, v[3:4]
+; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
-; GISEL-NEXT: v_mul_lo_u32 v4, v6, v7
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v8
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v14, v3
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v6, vcc
-; GISEL-NEXT: v_sub_i32_e64 v5, s[4:5], v2, v11
-; GISEL-NEXT: v_subbrev_u32_e64 v7, s[6:7], 0, v4, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v7, v6
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v6, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v5, v11
-; GISEL-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v14, v3, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v8
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v4, v5, v9, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v10
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v10
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v10, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v13, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v11, v2
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v10, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v10, v3
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v7
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v2, v6
+; GISEL-NEXT: v_subbrev_u32_e64 v9, s[4:5], 0, v3, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v6
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v7
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v12
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v12
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v12
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v12, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i64:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_mov_b32_e32 v9, v1
-; CGP-NEXT: v_mov_b32_e32 v8, v0
-; CGP-NEXT: v_or_b32_e32 v0, v9, v5
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT: v_mov_b32_e32 v11, v1
+; CGP-NEXT: v_mov_b32_e32 v10, v0
+; CGP-NEXT: v_or_b32_e32 v1, v11, v5
+; CGP-NEXT: v_mov_b32_e32 v0, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT: v_mov_b32_e32 v8, v2
+; CGP-NEXT: v_mov_b32_e32 v9, v3
; CGP-NEXT: ; implicit-def: $vgpr0_vgpr1
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CGP-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
; CGP-NEXT: s_cbranch_execz .LBB2_2
; CGP-NEXT: ; %bb.1:
-; CGP-NEXT: v_ashrrev_i32_e32 v0, 31, v5
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v4, v0
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v5, v0, vcc
-; CGP-NEXT: v_xor_b32_e32 v10, v4, v0
-; CGP-NEXT: v_xor_b32_e32 v11, v1, v0
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v11
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v10
-; CGP-NEXT: v_sub_i32_e32 v14, vcc, 0, v11
-; CGP-NEXT: v_subb_u32_e32 v15, vcc, 0, v10, vcc
-; CGP-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CGP-NEXT: v_rcp_f32_e32 v0, v0
-; CGP-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; CGP-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; CGP-NEXT: v_trunc_f32_e32 v1, v1
-; CGP-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; CGP-NEXT: v_cvt_u32_f32_e32 v12, v1
-; CGP-NEXT: v_cvt_u32_f32_e32 v13, v0
-; CGP-NEXT: v_mul_lo_u32 v4, v14, v12
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v5, v15, v13
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v1, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v16, 0
-; CGP-NEXT: v_mul_hi_u32 v17, v13, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v0, 0
-; CGP-NEXT: v_add_i32_e32 v17, vcc, v17, v4
-; CGP-NEXT: v_addc_u32_e32 v18, vcc, 0, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v16, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v17, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v18, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v0
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v4, v14, v12
-; CGP-NEXT: v_mul_lo_u32 v5, v15, v13
-; CGP-NEXT: v_mul_hi_u32 v15, v13, v0
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v1, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v14, 0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v0, 0
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v4
-; CGP-NEXT: v_addc_u32_e32 v16, vcc, 0, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v14, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v15, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v16, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v13, v0
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v1, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v13, 31, v9
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v8, v13
-; CGP-NEXT: v_xor_b32_e32 v8, v0, v13
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v9, v13, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v12, 0
-; CGP-NEXT: v_mul_hi_u32 v9, v8, v4
-; CGP-NEXT: v_xor_b32_e32 v14, v5, v13
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v4, 0
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v0
-; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v12, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v9, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v15, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; CGP-NEXT: v_mul_lo_u32 v5, v11, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v4, v10, v4
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v5
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v14, v1
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v8, v0
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v10, vcc
-; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v0, v11
-; CGP-NEXT: v_subbrev_u32_e64 v8, s[6:7], 0, v4, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v8, v10
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v11
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[6:7], v8, v10
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v10, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v9, v9, v12, s[6:7]
-; CGP-NEXT: v_sub_i32_e64 v12, s[4:5], v5, v11
-; CGP-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v14, v1, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v9
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v10
-; CGP-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v11
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v1, v10
-; CGP-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v4, v5, v12, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT: v_xor_b32_e32 v0, v0, v13
-; CGP-NEXT: v_xor_b32_e32 v1, v1, v13
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v13
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v13, vcc
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CGP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v5
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v1
+; CGP-NEXT: v_addc_u32_e32 v2, vcc, v5, v1, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v1
+; CGP-NEXT: v_xor_b32_e32 v1, v2, v1
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v0
+; CGP-NEXT: v_cvt_f32_u32_e32 v3, v1
+; CGP-NEXT: v_sub_i32_e32 v16, vcc, 0, v0
+; CGP-NEXT: v_subb_u32_e32 v17, vcc, 0, v1, vcc
+; CGP-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v2
+; CGP-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CGP-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
+; CGP-NEXT: v_trunc_f32_e32 v3, v3
+; CGP-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3
+; CGP-NEXT: v_cvt_u32_f32_e32 v15, v2
+; CGP-NEXT: v_cvt_u32_f32_e32 v14, v3
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v16, v15, 0
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v14, v[3:4]
+; CGP-NEXT: v_mul_hi_u32 v3, v15, v2
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v15, v[4:5]
+; CGP-NEXT: v_mul_lo_u32 v4, v14, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v14, v2
+; CGP-NEXT: v_mul_lo_u32 v5, v15, v12
+; CGP-NEXT: v_mul_hi_u32 v13, v15, v12
+; CGP-NEXT: v_mul_lo_u32 v18, v14, v12
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v13
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v18
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_mul_hi_u32 v5, v14, v12
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v2
+; CGP-NEXT: v_addc_u32_e32 v14, vcc, v14, v3, vcc
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v16, v15, 0
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v14, v[3:4]
+; CGP-NEXT: v_ashrrev_i32_e32 v16, 31, v11
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v10, v16
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v15, v[4:5]
+; CGP-NEXT: v_mul_hi_u32 v5, v15, v2
+; CGP-NEXT: v_xor_b32_e32 v13, v3, v16
+; CGP-NEXT: v_mul_lo_u32 v3, v14, v2
+; CGP-NEXT: v_mul_lo_u32 v10, v15, v12
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, v11, v16, vcc
+; CGP-NEXT: v_mul_hi_u32 v2, v14, v2
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v5, v15, v12
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v10, v3
+; CGP-NEXT: v_mul_lo_u32 v10, v14, v12
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_mul_hi_u32 v10, v14, v12
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v10, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v15, v2
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v14, v3, vcc
+; CGP-NEXT: v_mul_hi_u32 v5, v13, v2
+; CGP-NEXT: v_mul_lo_u32 v10, v13, v3
+; CGP-NEXT: v_xor_b32_e32 v12, v4, v16
+; CGP-NEXT: v_mul_lo_u32 v4, v12, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v12, v2
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v5, v13, v3
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v10, v4
+; CGP-NEXT: v_mul_lo_u32 v10, v12, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v2, v4
+; CGP-NEXT: v_mul_hi_u32 v10, v12, v3
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v14, 0
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v4
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v10, v[3:4]
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v13, v2
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v1, v14, v[4:5]
+; CGP-NEXT: v_subb_u32_e64 v3, s[4:5], v12, v10, vcc
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v12, v10
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v1
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v0
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v3, v1
+; CGP-NEXT: v_subb_u32_e32 v4, vcc, v4, v1, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[4:5]
+; CGP-NEXT: v_sub_i32_e32 v10, vcc, v2, v0
+; CGP-NEXT: v_subbrev_u32_e64 v11, s[4:5], 0, v4, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v1
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v0
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v11, v1
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v4, v1, vcc
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v10, v0
+; CGP-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[4:5]
+; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
+; CGP-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v11, v1, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v16
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v16
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v16
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v16, vcc
+; CGP-NEXT: ; implicit-def: $vgpr4
+; CGP-NEXT: ; implicit-def: $vgpr10
; CGP-NEXT: .LBB2_2: ; %Flow1
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
; CGP-NEXT: s_cbranch_execz .LBB2_4
; CGP-NEXT: ; %bb.3:
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v4
@@ -671,158 +775,185 @@ define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-NEXT: v_mul_lo_u32 v1, v1, v0
; CGP-NEXT: v_mul_hi_u32 v1, v0, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_mul_hi_u32 v0, v8, v0
+; CGP-NEXT: v_mul_hi_u32 v0, v10, v0
+; CGP-NEXT: v_mov_b32_e32 v1, 0
; CGP-NEXT: v_mul_lo_u32 v0, v0, v4
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v8, v0
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v10, v0
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v4
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v4
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CGP-NEXT: v_mov_b32_e32 v1, 0
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; CGP-NEXT: .LBB2_4: ; %.split
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_or_b32_e32 v4, v3, v7
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT: v_or_b32_e32 v3, v9, v7
+; CGP-NEXT: v_mov_b32_e32 v2, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CGP-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; CGP-NEXT: s_cbranch_execz .LBB2_6
-; CGP-NEXT: ; %bb.5:
-; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v7
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v7, v4, vcc
-; CGP-NEXT: v_xor_b32_e32 v9, v6, v4
-; CGP-NEXT: v_xor_b32_e32 v10, v5, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v4, v10
-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v9
-; CGP-NEXT: v_sub_i32_e32 v13, vcc, 0, v10
-; CGP-NEXT: v_subb_u32_e32 v14, vcc, 0, v9, vcc
-; CGP-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CGP-NEXT: v_rcp_f32_e32 v4, v4
+; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
+; CGP-NEXT: s_cbranch_execnz .LBB2_7
+; CGP-NEXT: ; %bb.5: ; %Flow
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_cbranch_execnz .LBB2_8
+; CGP-NEXT: .LBB2_6: ; %.split.split
+; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
+; CGP-NEXT: s_setpc_b64 s[30:31]
+; CGP-NEXT: .LBB2_7:
+; CGP-NEXT: v_ashrrev_i32_e32 v3, 31, v7
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v6, v3
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, v7, v3, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v2, v3
+; CGP-NEXT: v_xor_b32_e32 v3, v4, v3
+; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2
+; CGP-NEXT: v_cvt_f32_u32_e32 v5, v3
+; CGP-NEXT: v_sub_i32_e32 v14, vcc, 0, v2
+; CGP-NEXT: v_subb_u32_e32 v15, vcc, 0, v3, vcc
+; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
; CGP-NEXT: v_trunc_f32_e32 v5, v5
-; CGP-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CGP-NEXT: v_cvt_u32_f32_e32 v11, v5
-; CGP-NEXT: v_cvt_u32_f32_e32 v12, v4
-; CGP-NEXT: v_mul_lo_u32 v6, v13, v11
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; CGP-NEXT: v_mul_lo_u32 v7, v14, v12
+; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5
+; CGP-NEXT: v_cvt_u32_f32_e32 v13, v4
+; CGP-NEXT: v_cvt_u32_f32_e32 v12, v5
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v13, 0
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v14, v12, v[5:6]
+; CGP-NEXT: v_mul_hi_u32 v5, v13, v4
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v13, v[6:7]
+; CGP-NEXT: v_mul_lo_u32 v6, v12, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v12, v4
+; CGP-NEXT: v_mul_lo_u32 v7, v13, v10
+; CGP-NEXT: v_mul_hi_u32 v11, v13, v10
+; CGP-NEXT: v_mul_lo_u32 v16, v12, v10
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v15, 0
-; CGP-NEXT: v_mul_hi_u32 v16, v12, v4
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v16, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v11, v15, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v16, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v4
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, v11, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; CGP-NEXT: v_mul_lo_u32 v6, v13, v11
-; CGP-NEXT: v_mul_lo_u32 v7, v14, v12
-; CGP-NEXT: v_mul_hi_u32 v14, v12, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v13, 0
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v11, v13, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v14, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v11, v5, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v3, v7, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v6, 0
-; CGP-NEXT: v_mul_hi_u32 v11, v8, v4
-; CGP-NEXT: v_xor_b32_e32 v12, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v4, 0
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v2
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v3, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v6, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v11, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v13, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v2
-; CGP-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; CGP-NEXT: v_mul_lo_u32 v5, v10, v2
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v4, v9, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v12, v3
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v16
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT: v_mul_hi_u32 v7, v12, v10
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v4
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, v12, v5, vcc
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v13, 0
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v14, v12, v[5:6]
+; CGP-NEXT: v_ashrrev_i32_e32 v14, 31, v9
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v14
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v13, v[6:7]
+; CGP-NEXT: v_mul_hi_u32 v7, v13, v4
+; CGP-NEXT: v_xor_b32_e32 v11, v5, v14
+; CGP-NEXT: v_mul_lo_u32 v5, v12, v4
+; CGP-NEXT: v_mul_lo_u32 v8, v13, v10
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, v9, v14, vcc
+; CGP-NEXT: v_mul_hi_u32 v4, v12, v4
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v13, v10
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; CGP-NEXT: v_mul_lo_u32 v8, v12, v10
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_mul_hi_u32 v8, v12, v10
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v13, v4
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, v12, v5, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v11, v4
+; CGP-NEXT: v_mul_lo_u32 v8, v11, v5
+; CGP-NEXT: v_xor_b32_e32 v10, v6, v14
+; CGP-NEXT: v_mul_lo_u32 v6, v10, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v10, v4
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v11, v5
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT: v_mul_lo_u32 v8, v10, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v4, v6
+; CGP-NEXT: v_mul_hi_u32 v8, v10, v5
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v12, 0
+; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v6
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v2, v8, v[5:6]
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v11, v4
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v3, v12, v[6:7]
+; CGP-NEXT: v_subb_u32_e64 v5, s[4:5], v10, v8, vcc
+; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v10, v8
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v3
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v2
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v5, v3
+; CGP-NEXT: v_subb_u32_e32 v6, vcc, v6, v3, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[4:5]
+; CGP-NEXT: v_sub_i32_e32 v8, vcc, v4, v2
+; CGP-NEXT: v_subbrev_u32_e64 v9, s[4:5], 0, v6, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v3
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v2
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v3
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v6, v3, vcc
; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v9, vcc
-; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v2, v10
-; CGP-NEXT: v_subbrev_u32_e64 v6, s[6:7], 0, v4, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v9
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v10
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[6:7], v6, v9
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v9, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[6:7]
-; CGP-NEXT: v_sub_i32_e64 v11, s[4:5], v5, v10
-; CGP-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v3, vcc, v12, v3, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v8
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v9
-; CGP-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v10
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v9
-; CGP-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v4, v5, v11, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; CGP-NEXT: v_xor_b32_e32 v2, v2, v7
-; CGP-NEXT: v_xor_b32_e32 v3, v3, v7
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v7
-; CGP-NEXT: v_subb_u32_e32 v5, vcc, v3, v7, vcc
-; CGP-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CGP-NEXT: .LBB2_6: ; %Flow
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; CGP-NEXT: s_cbranch_execz .LBB2_8
-; CGP-NEXT: ; %bb.7:
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v6
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v6
-; CGP-NEXT: v_mov_b32_e32 v5, 0
-; CGP-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v4, v3
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; CGP-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v2, v14
+; CGP-NEXT: v_xor_b32_e32 v3, v3, v14
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v14
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v14, vcc
+; CGP-NEXT: ; implicit-def: $vgpr6
+; CGP-NEXT: ; implicit-def: $vgpr8
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_cbranch_execz .LBB2_6
+; CGP-NEXT: .LBB2_8:
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v6
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v2
+; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
-; CGP-NEXT: v_mul_lo_u32 v3, v3, v6
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v2, v8, v2
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_mul_lo_u32 v2, v2, v6
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v6
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v6
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc
-; CGP-NEXT: .LBB2_8: ; %.split.split
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_mov_b32_e32 v2, v4
-; CGP-NEXT: v_mov_b32_e32 v3, v5
; CGP-NEXT: s_setpc_b64 s[30:31]
%result = srem <2 x i64> %num, %den
ret <2 x i64> %result
@@ -832,37 +963,51 @@ define i64 @v_srem_i64_pow2k_denom(i64 %num) {
; CHECK-LABEL: v_srem_i64_pow2k_denom:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 20, v2
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v0, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; CHECK-NEXT: v_and_b32_e32 v2, 0xfffff000, v2
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; CHECK-NEXT: v_and_b32_e32 v1, 0xfff, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_and_b32_e32 v0, 0xfff, v0
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, vcc
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = srem i64 %num, 4096
ret i64 %result
}
define <2 x i64> @v_srem_v2i64_pow2k_denom(<2 x i64> %num) {
-; CHECK-LABEL: v_srem_v2i64_pow2k_denom:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; CHECK-NEXT: v_lshrrev_b32_e32 v4, 20, v4
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v0, v4
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; CHECK-NEXT: v_and_b32_e32 v4, 0xfffff000, v4
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; CHECK-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
-; CHECK-NEXT: v_lshrrev_b32_e32 v4, 20, v4
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v2, v4
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; CHECK-NEXT: v_and_b32_e32 v4, 0xfffff000, v4
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; CHECK-NEXT: v_subb_u32_e32 v3, vcc, v3, v5, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_srem_v2i64_pow2k_denom:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v3
+; GISEL-NEXT: v_and_b32_e32 v1, 0xfff, v1
+; GISEL-NEXT: v_and_b32_e32 v3, 0xfff, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT: v_and_b32_e32 v0, 0xfff, v0
+; GISEL-NEXT: v_and_b32_e32 v2, 0xfff, v2
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_v2i64_pow2k_denom:
+; CGP: ; %bb.0:
+; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; CGP-NEXT: v_and_b32_e32 v1, 0xfff, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT: v_and_b32_e32 v0, 0xfff, v0
+; CGP-NEXT: v_ashrrev_i32_e32 v3, 31, v3
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT: v_and_b32_e32 v3, 0xfff, v3
+; CGP-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_and_b32_e32 v2, 0xfff, v2
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, vcc
+; CGP-NEXT: s_setpc_b64 s[30:31]
%result = srem <2 x i64> %num, <i64 4096, i64 4096>
ret <2 x i64> %result
}
@@ -871,87 +1016,662 @@ define i64 @v_srem_i64_oddk_denom(i64 %num) {
; CHECK-LABEL: v_srem_i64_oddk_denom:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_mov_b32 s6, 0xfd81e19
-; CHECK-NEXT: v_mul_hi_u32 v2, v0, s6
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: s_mov_b32 s7, 0x6ca94220
-; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, s6, v[2:3]
-; CHECK-NEXT: v_mov_b32_e32 v2, v4
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, s7, v[2:3]
-; CHECK-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v5, v3
-; CHECK-NEXT: v_addc_u32_e64 v3, s[4:5], 0, 0, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v1, s7, v[2:3]
-; CHECK-NEXT: v_mul_lo_u32 v5, v4, s6
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, s6, v[2:3]
-; CHECK-NEXT: v_mul_lo_u32 v4, v4, s7
-; CHECK-NEXT: s_mov_b32 s6, 0x12d8fb
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_cvt_f32_u32_e32 v3, 0x12d8fb
+; CHECK-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
+; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; CHECK-NEXT: v_xor_b32_e32 v5, v0, v2
+; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; CHECK-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
+; CHECK-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
+; CHECK-NEXT: v_trunc_f32_e32 v4, v4
+; CHECK-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
+; CHECK-NEXT: v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v4
+; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, v3
+; CHECK-NEXT: v_readfirstlane_b32 s4, v3
+; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CHECK-NEXT: v_readfirstlane_b32 s5, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v3, s6
+; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: s_mul_i32 s8, s4, -1
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: v_readfirstlane_b32 s8, v6
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
+; CHECK-NEXT: v_mul_hi_u32 v6, v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v3, v3, s7
+; CHECK-NEXT: s_mul_i32 s10, s5, s6
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s8, s8, s10
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_add_i32 s9, s9, s6
+; CHECK-NEXT: v_readfirstlane_b32 s6, v6
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_mul_i32 s10, s5, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s8
+; CHECK-NEXT: s_cselect_b32 s8, 1, 0
+; CHECK-NEXT: s_add_u32 s6, s6, s10
+; CHECK-NEXT: s_cselect_b32 s10, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s7
+; CHECK-NEXT: s_add_i32 s8, s8, s10
+; CHECK-NEXT: s_add_u32 s6, s6, s9
+; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: s_add_i32 s8, s8, s7
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: s_add_i32 s7, s7, s8
+; CHECK-NEXT: s_add_u32 s4, s4, s6
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, s4
+; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CHECK-NEXT: s_addc_u32 s5, s5, s7
+; CHECK-NEXT: v_xor_b32_e32 v6, v1, v2
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
+; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: s_mul_i32 s8, s4, -1
+; CHECK-NEXT: s_add_u32 s7, s8, s7
+; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s7
+; CHECK-NEXT: s_mul_i32 s9, s4, s7
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v0
+; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
+; CHECK-NEXT: s_mul_i32 s6, s5, s6
+; CHECK-NEXT: s_add_u32 s8, s8, s9
+; CHECK-NEXT: s_cselect_b32 s9, 1, 0
+; CHECK-NEXT: s_add_u32 s6, s8, s6
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_add_i32 s9, s9, s6
+; CHECK-NEXT: v_readfirstlane_b32 s6, v0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: s_mul_i32 s7, s5, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s8
+; CHECK-NEXT: s_cselect_b32 s8, 1, 0
+; CHECK-NEXT: s_add_u32 s6, s6, s7
+; CHECK-NEXT: s_cselect_b32 s7, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, s5, v1
+; CHECK-NEXT: s_add_i32 s7, s8, s7
+; CHECK-NEXT: s_add_u32 s6, s6, s9
+; CHECK-NEXT: s_cselect_b32 s8, 1, 0
+; CHECK-NEXT: s_add_i32 s7, s7, s8
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: s_add_i32 s8, s8, s7
+; CHECK-NEXT: s_add_u32 s4, s4, s6
+; CHECK-NEXT: s_addc_u32 s5, s5, s8
+; CHECK-NEXT: v_mul_hi_u32 v0, v5, s4
+; CHECK-NEXT: v_mul_lo_u32 v1, v5, s5
+; CHECK-NEXT: v_mul_lo_u32 v3, v6, s4
+; CHECK-NEXT: v_mul_hi_u32 v4, v5, s5
+; CHECK-NEXT: v_mul_hi_u32 v9, v6, s5
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v3, v6, s4
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v6, s5
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT: v_ashr_i64 v[4:5], v[2:3], 19
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 31, v3
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, s6, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc
-; CHECK-NEXT: v_mad_i32_i24 v3, v4, s6, v3
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v1, v0
+; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v4, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v8
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v3
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v8, v[1:2]
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v5, v0
+; CHECK-NEXT: v_subb_u32_e64 v1, s[4:5], v6, v3, vcc
+; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_sub_i32_e32 v5, vcc, v0, v7
+; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v5, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v7
+; CHECK-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc
+; CHECK-NEXT: v_sub_i32_e32 v7, vcc, v5, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; CHECK-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v4, -1, v4, s[4:5]
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
+; CHECK-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = srem i64 %num, 1235195
ret i64 %result
}
define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
-; CHECK-LABEL: v_srem_v2i64_oddk_denom:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_mov_b32 s6, 0xfd81e19
-; CHECK-NEXT: v_mul_hi_u32 v4, v0, s6
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: s_mov_b32 s7, 0x6ca94220
-; CHECK-NEXT: s_mov_b32 s8, 0x12d8fb
-; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, s6, v[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
-; CHECK-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, s7, v[4:5]
-; CHECK-NEXT: v_mul_hi_u32 v4, v2, s6
-; CHECK-NEXT: v_ashrrev_i32_e32 v8, 31, v1
-; CHECK-NEXT: v_mul_lo_u32 v10, v8, s6
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v9
-; CHECK-NEXT: v_addc_u32_e64 v7, s[4:5], 0, 0, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, s7, v[6:7]
-; CHECK-NEXT: v_mul_lo_u32 v11, v8, s7
-; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, s6, v[6:7]
-; CHECK-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v3, s6, v[4:5]
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v10, v7
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v11, v4
-; CHECK-NEXT: v_mov_b32_e32 v4, v8
-; CHECK-NEXT: v_ashr_i64 v[10:11], v[6:7], 19
-; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, s7, v[4:5]
-; CHECK-NEXT: v_lshrrev_b32_e32 v6, 31, v7
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v10, v6
-; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v9, v5
-; CHECK-NEXT: v_addc_u32_e64 v5, s[4:5], 0, 0, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, s7, v[4:5]
-; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v6, s8, 0
-; CHECK-NEXT: v_ashrrev_i32_e32 v9, 31, v3
-; CHECK-NEXT: v_mul_lo_u32 v10, v9, s6
-; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v9, s6, v[4:5]
-; CHECK-NEXT: v_mad_i32_i24 v11, v8, s8, v7
-; CHECK-NEXT: v_mul_lo_u32 v7, v9, s7
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v10, v5
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; CHECK-NEXT: v_ashr_i64 v[7:8], v[4:5], 19
-; CHECK-NEXT: v_lshrrev_b32_e32 v4, 31, v5
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v7, v4
-; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v4, s8, 0
-; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v11, vcc
-; CHECK-NEXT: v_mad_i32_i24 v5, v7, s8, v5
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; CHECK-NEXT: v_subb_u32_e32 v3, vcc, v3, v5, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_srem_v2i64_oddk_denom:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v5, 0
+; GISEL-NEXT: s_mov_b32 s8, 1
+; GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
+; GISEL-NEXT: v_trunc_f32_e32 v6, v6
+; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
+; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v1, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v1, v0, v4
+; GISEL-NEXT: v_mov_b32_e32 v0, 0xffed2705
+; GISEL-NEXT: v_mul_hi_u32 v8, v0, v7
+; GISEL-NEXT: v_readfirstlane_b32 s7, v7
+; GISEL-NEXT: s_mul_i32 s5, s7, 0xffed2705
+; GISEL-NEXT: v_readfirstlane_b32 s6, v5
+; GISEL-NEXT: v_readfirstlane_b32 s9, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v7, s5
+; GISEL-NEXT: s_subb_u32 s4, 0, 0
+; GISEL-NEXT: s_mul_i32 s10, s6, 0xffed2705
+; GISEL-NEXT: s_add_u32 s9, s10, s9
+; GISEL-NEXT: s_mul_i32 s10, s4, s7
+; GISEL-NEXT: s_add_u32 s10, s10, s9
+; GISEL-NEXT: v_readfirstlane_b32 s11, v8
+; GISEL-NEXT: s_mul_i32 s12, s7, s10
+; GISEL-NEXT: v_mul_hi_u32 v8, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v9, v7, s10
+; GISEL-NEXT: s_mul_i32 s13, s6, s5
+; GISEL-NEXT: s_add_u32 s12, s11, s12
+; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: s_add_u32 s12, s12, s13
+; GISEL-NEXT: s_cselect_b32 s5, 1, 0
+; GISEL-NEXT: s_add_i32 s14, s14, s5
+; GISEL-NEXT: v_readfirstlane_b32 s12, v8
+; GISEL-NEXT: v_readfirstlane_b32 s5, v9
+; GISEL-NEXT: s_mul_i32 s15, s6, s10
+; GISEL-NEXT: s_add_u32 s5, s12, s5
+; GISEL-NEXT: s_cselect_b32 s16, 1, 0
+; GISEL-NEXT: s_add_u32 s5, s5, s15
+; GISEL-NEXT: s_cselect_b32 s15, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v8, v5, s10
+; GISEL-NEXT: s_add_i32 s15, s16, s15
+; GISEL-NEXT: s_add_u32 s5, s5, s14
+; GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; GISEL-NEXT: s_add_i32 s15, s15, s10
+; GISEL-NEXT: v_readfirstlane_b32 s10, v8
+; GISEL-NEXT: s_add_i32 s10, s10, s15
+; GISEL-NEXT: s_add_u32 s5, s7, s5
+; GISEL-NEXT: v_mul_hi_u32 v8, v0, s5
+; GISEL-NEXT: s_mul_i32 s14, s5, 0xffed2705
+; GISEL-NEXT: v_xor_b32_e32 v14, v6, v4
+; GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GISEL-NEXT: s_addc_u32 s10, s6, s10
+; GISEL-NEXT: v_readfirstlane_b32 s15, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, s5, v6
+; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
+; GISEL-NEXT: s_add_u32 s15, s16, s15
+; GISEL-NEXT: s_mul_i32 s4, s4, s5
+; GISEL-NEXT: s_add_u32 s4, s4, s15
+; GISEL-NEXT: v_readfirstlane_b32 s15, v8
+; GISEL-NEXT: v_mov_b32_e32 v8, s4
+; GISEL-NEXT: s_mul_i32 s16, s5, s4
+; GISEL-NEXT: v_mul_hi_u32 v6, s10, v6
+; GISEL-NEXT: v_mul_hi_u32 v9, s5, v8
+; GISEL-NEXT: s_mul_i32 s14, s10, s14
+; GISEL-NEXT: s_add_u32 s15, s15, s16
+; GISEL-NEXT: s_cselect_b32 s16, 1, 0
+; GISEL-NEXT: s_add_u32 s14, s15, s14
+; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: s_add_i32 s16, s16, s14
+; GISEL-NEXT: v_readfirstlane_b32 s14, v6
+; GISEL-NEXT: v_readfirstlane_b32 s15, v9
+; GISEL-NEXT: s_mul_i32 s4, s10, s4
+; GISEL-NEXT: s_add_u32 s14, s14, s15
+; GISEL-NEXT: s_cselect_b32 s15, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s14, s4
+; GISEL-NEXT: s_cselect_b32 s14, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v6, s10, v8
+; GISEL-NEXT: s_add_i32 s14, s15, s14
+; GISEL-NEXT: s_add_u32 s4, s4, s16
+; GISEL-NEXT: s_cselect_b32 s15, 1, 0
+; GISEL-NEXT: s_add_i32 s14, s14, s15
+; GISEL-NEXT: v_readfirstlane_b32 s15, v6
+; GISEL-NEXT: s_add_i32 s15, s15, s14
+; GISEL-NEXT: s_add_u32 s4, s5, s4
+; GISEL-NEXT: s_addc_u32 s5, s10, s15
+; GISEL-NEXT: v_mul_hi_u32 v8, v1, s4
+; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, s4
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
+; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v10, v14, s4
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v15, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v10
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v12, v[9:10]
+; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], 0, v15, v[10:11]
+; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], v14, v12, vcc
+; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v14, v12
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v10, -1, v10, s[4:5]
+; GISEL-NEXT: s_subb_u32 s4, 0, 0
+; GISEL-NEXT: s_mul_i32 s5, s4, s7
+; GISEL-NEXT: s_add_u32 s5, s5, s9
+; GISEL-NEXT: s_mul_i32 s8, s7, s5
+; GISEL-NEXT: v_mul_hi_u32 v7, v7, s5
+; GISEL-NEXT: s_add_u32 s8, s11, s8
+; GISEL-NEXT: s_cselect_b32 s9, 1, 0
+; GISEL-NEXT: s_add_u32 s8, s8, s13
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: s_add_i32 s9, s9, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: s_mul_i32 s10, s6, s5
+; GISEL-NEXT: s_add_u32 s8, s12, s8
+; GISEL-NEXT: s_cselect_b32 s11, 1, 0
+; GISEL-NEXT: s_add_u32 s8, s8, s10
+; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v5, v5, s5
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v1, v6
+; GISEL-NEXT: s_add_i32 s10, s11, s10
+; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT: s_add_u32 s5, s8, s9
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v6
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9
+; GISEL-NEXT: s_add_i32 s10, s10, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v12, -1, v12, vcc
+; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v11, v6
+; GISEL-NEXT: s_add_i32 s8, s8, s10
+; GISEL-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v9, vcc
+; GISEL-NEXT: s_add_u32 s5, s7, s5
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT: v_mul_hi_u32 v0, v0, s5
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v14, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
+; GISEL-NEXT: v_xor_b32_e32 v11, v8, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3
+; GISEL-NEXT: s_mul_i32 s7, s5, 0xffed2705
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT: s_addc_u32 s6, s6, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, s7
+; GISEL-NEXT: v_xor_b32_e32 v5, v2, v8
+; GISEL-NEXT: v_mul_hi_u32 v2, s5, v0
+; GISEL-NEXT: s_mul_i32 s9, s6, 0xffed2705
+; GISEL-NEXT: s_add_u32 s8, s9, s8
+; GISEL-NEXT: s_mul_i32 s4, s4, s5
+; GISEL-NEXT: s_add_u32 s4, s4, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v2
+; GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-NEXT: s_mul_i32 s9, s5, s4
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
+; GISEL-NEXT: v_mul_hi_u32 v7, s5, v2
+; GISEL-NEXT: s_mul_i32 s7, s6, s7
+; GISEL-NEXT: s_add_u32 s8, s8, s9
+; GISEL-NEXT: s_cselect_b32 s9, 1, 0
+; GISEL-NEXT: s_add_u32 s7, s8, s7
+; GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-NEXT: s_add_i32 s9, s9, s7
+; GISEL-NEXT: v_readfirstlane_b32 s7, v0
+; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: s_mul_i32 s4, s6, s4
+; GISEL-NEXT: s_add_u32 s7, s7, s8
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: s_add_u32 s4, s7, s4
+; GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
+; GISEL-NEXT: s_add_i32 s7, s8, s7
+; GISEL-NEXT: s_add_u32 s4, s4, s9
+; GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GISEL-NEXT: s_add_i32 s7, s7, s8
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GISEL-NEXT: s_add_i32 s8, s8, s7
+; GISEL-NEXT: s_add_u32 s4, s5, s4
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: s_addc_u32 s5, s6, s8
+; GISEL-NEXT: v_mul_hi_u32 v0, v5, s4
+; GISEL-NEXT: v_mul_lo_u32 v2, v5, s5
+; GISEL-NEXT: v_xor_b32_e32 v7, v3, v8
+; GISEL-NEXT: v_mul_lo_u32 v3, v7, s4
+; GISEL-NEXT: v_mul_hi_u32 v9, v5, s5
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v3, v7, s4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GISEL-NEXT: v_mul_lo_u32 v2, v7, s5
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v3
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v2, v0
+; GISEL-NEXT: v_mul_hi_u32 v10, v7, s5
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v12, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v12, v[9:10]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v7, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v7, v3
+; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_sub_i32_e32 v7, vcc, v2, v6
+; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v7, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v9, -1, v9, vcc
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4
+; GISEL-NEXT: v_subbrev_u32_e32 v10, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, -1, v5, s[4:5]
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v10, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v8
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v8
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_v2i64_oddk_denom:
+; CGP: ; %bb.0:
+; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
+; CGP-NEXT: v_cvt_f32_ubyte0_e32 v6, 0
+; CGP-NEXT: v_ashrrev_i32_e32 v5, 31, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT: v_xor_b32_e32 v9, v0, v5
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT: v_trunc_f32_e32 v6, v6
+; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
+; CGP-NEXT: v_cvt_u32_f32_e32 v7, v4
+; CGP-NEXT: v_mov_b32_e32 v4, 0xffed2705
+; CGP-NEXT: v_cvt_u32_f32_e32 v0, v6
+; CGP-NEXT: v_xor_b32_e32 v10, v1, v5
+; CGP-NEXT: v_mul_hi_u32 v6, v4, v7
+; CGP-NEXT: v_readfirstlane_b32 s6, v7
+; CGP-NEXT: s_mul_i32 s4, s6, 0xffed2705
+; CGP-NEXT: v_readfirstlane_b32 s7, v0
+; CGP-NEXT: v_readfirstlane_b32 s5, v6
+; CGP-NEXT: v_mul_hi_u32 v6, v7, s4
+; CGP-NEXT: s_mul_i32 s8, s7, 0xffed2705
+; CGP-NEXT: s_add_u32 s5, s8, s5
+; CGP-NEXT: s_mul_i32 s8, s6, -1
+; CGP-NEXT: s_add_u32 s5, s8, s5
+; CGP-NEXT: v_readfirstlane_b32 s8, v6
+; CGP-NEXT: s_mul_i32 s9, s6, s5
+; CGP-NEXT: v_mul_hi_u32 v6, v0, s4
+; CGP-NEXT: v_mul_hi_u32 v7, v7, s5
+; CGP-NEXT: s_mul_i32 s10, s7, s4
+; CGP-NEXT: s_add_u32 s11, s8, s9
+; CGP-NEXT: s_cselect_b32 s12, 1, 0
+; CGP-NEXT: s_add_u32 s11, s11, s10
+; CGP-NEXT: s_cselect_b32 s4, 1, 0
+; CGP-NEXT: s_add_i32 s12, s12, s4
+; CGP-NEXT: v_readfirstlane_b32 s11, v6
+; CGP-NEXT: v_readfirstlane_b32 s13, v7
+; CGP-NEXT: s_mul_i32 s14, s7, s5
+; CGP-NEXT: s_add_u32 s4, s11, s13
+; CGP-NEXT: s_cselect_b32 s15, 1, 0
+; CGP-NEXT: s_add_u32 s4, s4, s14
+; CGP-NEXT: s_cselect_b32 s16, 1, 0
+; CGP-NEXT: v_mul_hi_u32 v0, v0, s5
+; CGP-NEXT: s_add_i32 s15, s15, s16
+; CGP-NEXT: s_add_u32 s4, s4, s12
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
+; CGP-NEXT: s_add_i32 s15, s15, s5
+; CGP-NEXT: v_readfirstlane_b32 s12, v0
+; CGP-NEXT: s_add_i32 s5, s12, s15
+; CGP-NEXT: s_add_u32 s4, s6, s4
+; CGP-NEXT: v_mul_hi_u32 v0, v4, s4
+; CGP-NEXT: s_mul_i32 s15, s4, 0xffed2705
+; CGP-NEXT: s_addc_u32 s5, s7, s5
+; CGP-NEXT: s_mul_i32 s17, s5, 0xffed2705
+; CGP-NEXT: v_readfirstlane_b32 s16, v0
+; CGP-NEXT: v_mov_b32_e32 v0, s15
+; CGP-NEXT: v_mul_hi_u32 v1, s4, v0
+; CGP-NEXT: s_add_u32 s16, s17, s16
+; CGP-NEXT: s_mul_i32 s17, s4, -1
+; CGP-NEXT: s_add_u32 s16, s17, s16
+; CGP-NEXT: v_readfirstlane_b32 s17, v1
+; CGP-NEXT: v_mov_b32_e32 v1, s16
+; CGP-NEXT: s_mul_i32 s18, s4, s16
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v0
+; CGP-NEXT: v_mul_hi_u32 v6, s4, v1
+; CGP-NEXT: s_mul_i32 s15, s5, s15
+; CGP-NEXT: s_add_u32 s17, s17, s18
+; CGP-NEXT: s_cselect_b32 s18, 1, 0
+; CGP-NEXT: s_add_u32 s15, s17, s15
+; CGP-NEXT: s_cselect_b32 s15, 1, 0
+; CGP-NEXT: s_add_i32 s18, s18, s15
+; CGP-NEXT: v_readfirstlane_b32 s15, v0
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
+; CGP-NEXT: s_mul_i32 s16, s5, s16
+; CGP-NEXT: s_add_u32 s15, s15, s17
+; CGP-NEXT: s_cselect_b32 s17, 1, 0
+; CGP-NEXT: s_add_u32 s15, s15, s16
+; CGP-NEXT: s_cselect_b32 s16, 1, 0
+; CGP-NEXT: v_mul_hi_u32 v0, s5, v1
+; CGP-NEXT: s_add_i32 s16, s17, s16
+; CGP-NEXT: s_add_u32 s15, s15, s18
+; CGP-NEXT: s_cselect_b32 s17, 1, 0
+; CGP-NEXT: s_add_i32 s16, s16, s17
+; CGP-NEXT: v_readfirstlane_b32 s17, v0
+; CGP-NEXT: s_add_i32 s17, s17, s16
+; CGP-NEXT: s_add_u32 s4, s4, s15
+; CGP-NEXT: s_addc_u32 s5, s5, s17
+; CGP-NEXT: v_mul_hi_u32 v0, v9, s4
+; CGP-NEXT: v_mul_lo_u32 v1, v9, s5
+; CGP-NEXT: v_mul_lo_u32 v7, v10, s4
+; CGP-NEXT: v_mul_hi_u32 v8, v9, s5
+; CGP-NEXT: v_mov_b32_e32 v6, 0x12d8fb
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v7
+; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v10, s4
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CGP-NEXT: v_mul_lo_u32 v1, v10, s5
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v7, v1
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v1, v0
+; CGP-NEXT: v_mul_hi_u32 v12, v10, s5
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v8, 0
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v11
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v7
+; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v6, v11, v[1:2]
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v9, v0
+; CGP-NEXT: v_subb_u32_e64 v1, s[4:5], v10, v7, vcc
+; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v10, v7
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v6
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; CGP-NEXT: v_cndmask_b32_e64 v8, -1, v8, s[4:5]
+; CGP-NEXT: s_add_u32 s4, s8, s9
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
+; CGP-NEXT: s_add_u32 s4, s4, s10
+; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: s_cselect_b32 s4, 1, 0
+; CGP-NEXT: v_sub_i32_e32 v9, vcc, v0, v6
+; CGP-NEXT: s_add_i32 s5, s5, s4
+; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: s_add_u32 s4, s11, s13
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v9, v6
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_u32 s4, s4, s14
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; CGP-NEXT: s_cselect_b32 s9, 1, 0
+; CGP-NEXT: v_cndmask_b32_e32 v10, -1, v10, vcc
+; CGP-NEXT: v_sub_i32_e32 v11, vcc, v9, v6
+; CGP-NEXT: s_add_i32 s8, s8, s9
+; CGP-NEXT: v_subbrev_u32_e32 v12, vcc, 0, v7, vcc
+; CGP-NEXT: s_add_u32 s4, s4, s5
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
+; CGP-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
+; CGP-NEXT: s_add_i32 s8, s8, s5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
+; CGP-NEXT: s_add_i32 s12, s12, s8
+; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
+; CGP-NEXT: s_add_u32 s4, s6, s4
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
+; CGP-NEXT: v_mul_hi_u32 v4, v4, s4
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
+; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
+; CGP-NEXT: v_mov_b32_e32 v2, s6
+; CGP-NEXT: s_addc_u32 s5, s7, s12
+; CGP-NEXT: v_readfirstlane_b32 s7, v4
+; CGP-NEXT: v_mul_hi_u32 v4, s4, v2
+; CGP-NEXT: s_mul_i32 s8, s5, 0xffed2705
+; CGP-NEXT: s_add_u32 s7, s8, s7
+; CGP-NEXT: s_mul_i32 s8, s4, -1
+; CGP-NEXT: s_add_u32 s7, s8, s7
+; CGP-NEXT: v_readfirstlane_b32 s8, v4
+; CGP-NEXT: v_mov_b32_e32 v4, s7
+; CGP-NEXT: s_mul_i32 s9, s4, s7
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
+; CGP-NEXT: v_mul_hi_u32 v5, s4, v4
+; CGP-NEXT: s_mul_i32 s6, s5, s6
+; CGP-NEXT: s_add_u32 s8, s8, s9
+; CGP-NEXT: s_cselect_b32 s9, 1, 0
+; CGP-NEXT: s_add_u32 s6, s8, s6
+; CGP-NEXT: s_cselect_b32 s6, 1, 0
+; CGP-NEXT: s_add_i32 s9, s9, s6
+; CGP-NEXT: v_readfirstlane_b32 s6, v2
+; CGP-NEXT: v_readfirstlane_b32 s8, v5
+; CGP-NEXT: s_mul_i32 s7, s5, s7
+; CGP-NEXT: s_add_u32 s6, s6, s8
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_u32 s6, s6, s7
+; CGP-NEXT: s_cselect_b32 s7, 1, 0
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v4
+; CGP-NEXT: s_add_i32 s7, s8, s7
+; CGP-NEXT: s_add_u32 s6, s6, s9
+; CGP-NEXT: s_cselect_b32 s8, 1, 0
+; CGP-NEXT: s_add_i32 s7, s7, s8
+; CGP-NEXT: v_readfirstlane_b32 s8, v2
+; CGP-NEXT: s_add_i32 s8, s8, s7
+; CGP-NEXT: s_add_u32 s4, s4, s6
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT: s_addc_u32 s5, s5, s8
+; CGP-NEXT: v_mul_hi_u32 v2, v8, s4
+; CGP-NEXT: v_mul_lo_u32 v4, v8, s5
+; CGP-NEXT: v_xor_b32_e32 v9, v3, v7
+; CGP-NEXT: v_mul_lo_u32 v3, v9, s4
+; CGP-NEXT: v_mul_hi_u32 v5, v8, s5
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v3, v9, s4
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2
+; CGP-NEXT: v_mul_lo_u32 v4, v9, s5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v3, v2
+; CGP-NEXT: v_mul_hi_u32 v11, v9, s5
+; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v5, 0
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v4
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v10, v[3:4]
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
+; CGP-NEXT: v_subb_u32_e64 v3, s[4:5], v9, v4, vcc
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v9, v4
+; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; CGP-NEXT: v_sub_i32_e32 v8, vcc, v2, v6
+; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v8, v6
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; CGP-NEXT: v_cndmask_b32_e32 v9, -1, v9, vcc
+; CGP-NEXT: v_sub_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
+; CGP-NEXT: v_subbrev_u32_e32 v10, vcc, 0, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v5, -1, v5, s[4:5]
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; CGP-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v2, v7
+; CGP-NEXT: v_xor_b32_e32 v3, v3, v7
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT: s_setpc_b64 s[30:31]
%result = srem <2 x i64> %num, <i64 1235195, i64 1235195>
ret <2 x i64> %result
}
@@ -960,148 +1680,178 @@ define i64 @v_srem_i64_pow2_shl_denom(i64 %x, i64 %y) {
; CHECK-LABEL: v_srem_i64_pow2_shl_denom:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_mov_b64 s[4:5], 0x1000
-; CHECK-NEXT: v_lshl_b64 v[4:5], s[4:5], v2
-; CHECK-NEXT: v_or_b32_e32 v2, v1, v5
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT: v_mov_b32_e32 v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v4, v1
+; CHECK-NEXT: v_mov_b32_e32 v0, 0x1000
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_lshl_b64 v[5:6], v[0:1], v2
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: v_or_b32_e32 v1, v4, v6
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CHECK-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB7_2
-; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v5
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v5, v2, vcc
-; CHECK-NEXT: v_xor_b32_e32 v7, v4, v2
-; CHECK-NEXT: v_xor_b32_e32 v8, v3, v2
-; CHECK-NEXT: v_cvt_f32_u32_e32 v2, v8
-; CHECK-NEXT: v_cvt_f32_u32_e32 v3, v7
-; CHECK-NEXT: v_sub_i32_e32 v11, vcc, 0, v8
-; CHECK-NEXT: v_subb_u32_e32 v12, vcc, 0, v7, vcc
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
-; CHECK-NEXT: v_rcp_f32_e32 v2, v2
+; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB7_3
+; CHECK-NEXT: ; %bb.1: ; %Flow
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT: s_cbranch_execnz .LBB7_4
+; CHECK-NEXT: .LBB7_2: ; %.split
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; CHECK-NEXT: .LBB7_3:
+; CHECK-NEXT: v_ashrrev_i32_e32 v1, 31, v6
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v5, v1
+; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v6, v1, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1
+; CHECK-NEXT: v_xor_b32_e32 v1, v2, v1
+; CHECK-NEXT: v_cvt_f32_u32_e32 v2, v0
+; CHECK-NEXT: v_cvt_f32_u32_e32 v5, v1
+; CHECK-NEXT: v_sub_i32_e32 v12, vcc, 0, v0
+; CHECK-NEXT: v_subb_u32_e32 v13, vcc, 0, v1, vcc
+; CHECK-NEXT: v_mac_f32_e32 v2, 0x4f800000, v5
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v2, v2
; CHECK-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; CHECK-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
-; CHECK-NEXT: v_trunc_f32_e32 v3, v3
-; CHECK-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
-; CHECK-NEXT: v_cvt_u32_f32_e32 v9, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v10, v2
-; CHECK-NEXT: v_mul_lo_u32 v4, v11, v9
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v10, 0
-; CHECK-NEXT: v_mul_lo_u32 v5, v12, v10
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v13, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v13, 0
-; CHECK-NEXT: v_mul_hi_u32 v14, v10, v2
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v14, vcc, v14, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v13, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v14, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v2
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, v9, v3, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v10, 0
-; CHECK-NEXT: v_mul_lo_u32 v4, v11, v9
-; CHECK-NEXT: v_mul_lo_u32 v5, v12, v10
-; CHECK-NEXT: v_mul_hi_u32 v12, v10, v2
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v11, 0
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v12, vcc, v12, v3
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v11, 0
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v12, v5
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v10, v2
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v9, v3, vcc
-; CHECK-NEXT: v_ashrrev_i32_e32 v5, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_xor_b32_e32 v6, v0, v5
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v1, v5, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
-; CHECK-NEXT: v_mul_hi_u32 v9, v6, v2
-; CHECK-NEXT: v_xor_b32_e32 v10, v3, v5
-; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v2, 0
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v0
-; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v1, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v4, 0
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v9, v2
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v0
-; CHECK-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; CHECK-NEXT: v_mul_lo_u32 v3, v8, v0
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v2, 0
-; CHECK-NEXT: v_mul_lo_u32 v2, v7, v2
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v10, v1
+; CHECK-NEXT: v_mul_f32_e32 v5, 0x2f800000, v2
+; CHECK-NEXT: v_trunc_f32_e32 v5, v5
+; CHECK-NEXT: v_mac_f32_e32 v2, 0xcf800000, v5
+; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT: v_cvt_u32_f32_e32 v11, v5
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v2, 0
+; CHECK-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v11, v[6:7]
+; CHECK-NEXT: v_mul_hi_u32 v6, v2, v5
+; CHECK-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v2, v[7:8]
+; CHECK-NEXT: v_mul_lo_u32 v7, v11, v5
+; CHECK-NEXT: v_mul_hi_u32 v5, v11, v5
+; CHECK-NEXT: v_mul_lo_u32 v8, v2, v9
+; CHECK-NEXT: v_mul_hi_u32 v10, v2, v9
+; CHECK-NEXT: v_mul_lo_u32 v14, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v14
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT: v_mul_hi_u32 v8, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT: v_addc_u32_e32 v11, vcc, v11, v6, vcc
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v2, 0
+; CHECK-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v11, v[6:7]
+; CHECK-NEXT: v_ashrrev_i32_e32 v12, 31, v4
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v12
+; CHECK-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v2, v[7:8]
+; CHECK-NEXT: v_mul_hi_u32 v6, v2, v5
+; CHECK-NEXT: v_xor_b32_e32 v8, v3, v12
+; CHECK-NEXT: v_mul_lo_u32 v3, v11, v5
+; CHECK-NEXT: v_mul_lo_u32 v7, v2, v9
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v4, v12, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v11, v5
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v2, v9
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v7, v3
+; CHECK-NEXT: v_mul_lo_u32 v7, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_mul_hi_u32 v7, v11, v9
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v11, v5, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v8, v2
+; CHECK-NEXT: v_mul_lo_u32 v6, v8, v3
+; CHECK-NEXT: v_xor_b32_e32 v9, v4, v12
+; CHECK-NEXT: v_mul_lo_u32 v4, v9, v2
+; CHECK-NEXT: v_mul_hi_u32 v2, v9, v2
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v8, v3
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT: v_mul_lo_u32 v6, v9, v3
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v2, v4
+; CHECK-NEXT: v_mul_hi_u32 v6, v9, v3
+; CHECK-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v10, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v4
+; CHECK-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v6, v[3:4]
+; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
+; CHECK-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v10, v[4:5]
+; CHECK-NEXT: v_subb_u32_e64 v3, s[4:5], v9, v6, vcc
+; CHECK-NEXT: v_sub_i32_e64 v4, s[4:5], v9, v6
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v0
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v3, v1
+; CHECK-NEXT: v_subb_u32_e32 v4, vcc, v4, v1, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[4:5]
+; CHECK-NEXT: v_sub_i32_e32 v6, vcc, v2, v0
+; CHECK-NEXT: v_subbrev_u32_e64 v7, s[4:5], 0, v4, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v1
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v0
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v1
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v4, v1, vcc
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v6, v0
-; CHECK-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v7, vcc
-; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v8
-; CHECK-NEXT: v_subbrev_u32_e64 v4, s[6:7], 0, v2, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[6:7], v4, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[6:7]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v7
-; CHECK-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v7, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[6:7]
-; CHECK-NEXT: v_sub_i32_e64 v9, s[4:5], v3, v8
-; CHECK-NEXT: v_subbrev_u32_e64 v2, s[4:5], 0, v2, s[4:5]
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v10, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v1, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v1, v7
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v2, v3, v9, s[4:5]
+; CHECK-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[4:5]
+; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v12
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v12
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v12
+; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v12, vcc
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr3
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT: s_cbranch_execz .LBB7_2
+; CHECK-NEXT: .LBB7_4:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, v5
+; CHECK-NEXT: v_sub_i32_e32 v1, vcc, 0, v5
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v1, v0
+; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_mul_hi_u32 v0, v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mul_lo_u32 v0, v0, v5
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v3, v0
+; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v0, v5
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; CHECK-NEXT: v_xor_b32_e32 v0, v0, v5
-; CHECK-NEXT: v_xor_b32_e32 v1, v1, v5
; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v0, v5
-; CHECK-NEXT: v_subb_u32_e32 v3, vcc, v1, v5, vcc
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: .LBB7_2: ; %Flow
-; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; CHECK-NEXT: s_cbranch_execz .LBB7_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, v4
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, 0, v4
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_lo_u32 v2, v2, v1
-; CHECK-NEXT: v_mul_hi_u32 v2, v1, v2
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
-; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_mul_lo_u32 v1, v1, v4
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v0, v4
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v0, v4
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CHECK-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
-; CHECK-NEXT: .LBB7_4: ; %.split
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v0, v2
-; CHECK-NEXT: v_mov_b32_e32 v1, v3
; CHECK-NEXT: s_setpc_b64 s[30:31]
%shl.y = shl i64 4096, %y
%r = srem i64 %x, %shl.y
@@ -1112,345 +1862,423 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-LABEL: v_srem_v2i64_pow2_shl_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b64 s[4:5], 0x1000
-; GISEL-NEXT: v_lshl_b64 v[4:5], s[4:5], v4
+; GISEL-NEXT: v_mov_b32_e32 v10, 0x1000
+; GISEL-NEXT: v_mov_b32_e32 v11, 0
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[10:11], v4
; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v5
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v7
-; GISEL-NEXT: v_xor_b32_e32 v7, v4, v7
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v7
-; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v5
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 0, v7
-; GISEL-NEXT: v_subb_u32_e32 v15, vcc, 0, v5, vcc
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v5, v7, vcc
+; GISEL-NEXT: v_xor_b32_e32 v5, v4, v7
+; GISEL-NEXT: v_xor_b32_e32 v7, v8, v7
+; GISEL-NEXT: v_cvt_f32_u32_e32 v4, v5
+; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v7
+; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 0, v5
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, 0, v7, vcc
; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v8
-; GISEL-NEXT: v_rcp_f32_e32 v4, v4
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v4
; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
; GISEL-NEXT: v_mul_f32_e32 v8, 0x2f800000, v4
; GISEL-NEXT: v_trunc_f32_e32 v8, v8
; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v8
; GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v8
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v14, v4, 0
-; GISEL-NEXT: v_mul_lo_u32 v10, v14, v13
-; GISEL-NEXT: v_mul_lo_u32 v11, v15, v4
-; GISEL-NEXT: v_mul_hi_u32 v17, v4, v8
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v9, v11
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[6:7], v4, v16, 0
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v13, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v17, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v13, v16, 0
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v17, v11
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v12, vcc
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v13, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v14, v4, 0
-; GISEL-NEXT: v_mul_lo_u32 v10, v14, v13
-; GISEL-NEXT: v_mul_lo_u32 v11, v15, v4
-; GISEL-NEXT: v_mul_hi_u32 v15, v4, v8
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v9, v11
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[6:7], v4, v14, 0
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v13, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v13, v14, 0
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v15, v11
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v10, v12, vcc
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v4, v8
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, v13, v9, vcc
+; GISEL-NEXT: v_cvt_u32_f32_e32 v16, v8
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v4, 0
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
+; GISEL-NEXT: v_mul_hi_u32 v9, v4, v8
+; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v4, v[12:13]
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
+; GISEL-NEXT: v_mul_lo_u32 v13, v4, v14
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v12, v4, v14
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
+; GISEL-NEXT: v_mul_lo_u32 v13, v16, v14
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT: v_mul_hi_u32 v13, v16, v14
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v12, v9
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v13, v9
+; GISEL-NEXT: v_add_i32_e32 v19, vcc, v4, v8
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v16, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v17, v19, 0
; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_xor_b32_e32 v11, v0, v4
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v1, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v11, v10, 0
-; GISEL-NEXT: v_mul_hi_u32 v12, v11, v8
-; GISEL-NEXT: v_xor_b32_e32 v13, v9, v4
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v13, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v0
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v10, 0
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v12, v8
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v14, v9, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v8, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v12, v7, v0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v7, v10, 0
-; GISEL-NEXT: v_lshl_b64 v[8:9], s[4:5], v6
-; GISEL-NEXT: v_mul_lo_u32 v6, v5, v10
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v12
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v6
-; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v13, v1
-; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v11, v0
-; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], v6, v5, vcc
-; GISEL-NEXT: v_sub_i32_e64 v11, s[4:5], v10, v7
-; GISEL-NEXT: v_subbrev_u32_e64 v6, s[6:7], 0, v0, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v11, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v6, v5
-; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], v0, v5, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, v14, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v14, s[4:5], v11, v7
-; GISEL-NEXT: v_subbrev_u32_e64 v0, s[4:5], 0, v0, s[4:5]
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v12
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v13, v1, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v12, 31, v9
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[9:10]
+; GISEL-NEXT: v_mul_hi_u32 v9, v19, v8
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v19, v[12:13]
+; GISEL-NEXT: v_xor_b32_e32 v15, v0, v4
+; GISEL-NEXT: v_mul_lo_u32 v0, v16, v8
+; GISEL-NEXT: v_mul_hi_u32 v8, v16, v8
+; GISEL-NEXT: v_mul_lo_u32 v12, v19, v14
+; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], v6
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v19, v14
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
+; GISEL-NEXT: v_mul_lo_u32 v12, v16, v14
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v0, v6, v0, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v9, v12, vcc
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v12
-; GISEL-NEXT: v_xor_b32_e32 v9, v8, v12
-; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v9
-; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v6
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v10, v7
-; GISEL-NEXT: v_mac_f32_e32 v8, 0x4f800000, v12
-; GISEL-NEXT: v_rcp_f32_e32 v8, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v13, v7, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v1, v0, vcc
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_mul_hi_u32 v12, v16, v14
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v19, v0
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v16, v8, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v0
+; GISEL-NEXT: v_mul_lo_u32 v12, v15, v8
+; GISEL-NEXT: v_xor_b32_e32 v14, v1, v4
+; GISEL-NEXT: v_mul_lo_u32 v1, v14, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v15, v8
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
+; GISEL-NEXT: v_mul_lo_u32 v12, v14, v8
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v0, v1
+; GISEL-NEXT: v_mul_hi_u32 v8, v14, v8
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v16, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v8, v9
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v12, v[1:2]
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v7, v16, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v15, v0
+; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], v14, v12, vcc
+; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v14, v12
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v15, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v13, v5
+; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v0, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v15, v7
+; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v13, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v12, v1, v6, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e64 v16, s[4:5], 0, v0, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v11
+; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v10, v1
+; GISEL-NEXT: v_addc_u32_e64 v8, s[4:5], v11, v1, s[4:5]
+; GISEL-NEXT: v_xor_b32_e32 v6, v6, v1
+; GISEL-NEXT: v_xor_b32_e32 v8, v8, v1
+; GISEL-NEXT: v_cvt_f32_u32_e32 v1, v6
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v8
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v16, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v14, v5
+; GISEL-NEXT: v_mac_f32_e32 v1, 0x4f800000, v9
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v1, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v7
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v0, v7, vcc
+; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v1
; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v1, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
-; GISEL-NEXT: v_cvt_u32_f32_e32 v12, v0
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v1
-; GISEL-NEXT: v_sub_i32_e64 v15, s[6:7], 0, v9
-; GISEL-NEXT: v_subb_u32_e64 v16, s[6:7], 0, v6, s[6:7]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v15, v12, 0
-; GISEL-NEXT: v_mul_lo_u32 v7, v15, v13
-; GISEL-NEXT: v_mul_lo_u32 v8, v16, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, v14, s[4:5]
-; GISEL-NEXT: v_mul_hi_u32 v17, v12, v0
-; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v1, v7
-; GISEL-NEXT: v_add_i32_e64 v14, s[4:5], v1, v8
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v14, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v0, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v11, vcc
-; GISEL-NEXT: v_xor_b32_e32 v10, v10, v4
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v17, v7
-; GISEL-NEXT: v_addc_u32_e32 v17, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v14, 0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v17, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v8, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v0
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, v13, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v11, 0
-; GISEL-NEXT: v_mul_lo_u32 v7, v15, v12
-; GISEL-NEXT: v_mul_lo_u32 v8, v16, v11
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
-; GISEL-NEXT: v_mul_hi_u32 v14, v11, v0
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v7
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v1, v8
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v13, 0
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v0, 0
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v7
-; GISEL-NEXT: v_addc_u32_e32 v15, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v13, 0
+; GISEL-NEXT: v_trunc_f32_e32 v9, v1
+; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v9
+; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v0
+; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 0, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v17, 0
+; GISEL-NEXT: v_cvt_u32_f32_e32 v20, v9
+; GISEL-NEXT: v_subb_u32_e32 v19, vcc, 0, v8, vcc
+; GISEL-NEXT: v_mul_hi_u32 v21, v17, v0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v18, v20, v[1:2]
+; GISEL-NEXT: v_mul_lo_u32 v22, v20, v0
+; GISEL-NEXT: v_mul_hi_u32 v23, v20, v0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v14, v5
+; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v7, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v14, v0, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v19, v17, v[9:10]
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc
+; GISEL-NEXT: v_mul_lo_u32 v1, v17, v0
+; GISEL-NEXT: v_mul_hi_u32 v10, v17, v0
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v21, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v22
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
+; GISEL-NEXT: v_mul_lo_u32 v9, v20, v0
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v23, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT: v_mul_hi_u32 v0, v20, v0
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v17, v1
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, v20, v0, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v18, v14, 0
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v13, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v15, v5, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v18, v16, v[1:2]
+; GISEL-NEXT: v_xor_b32_e32 v1, v7, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v19, v14, v[9:10]
+; GISEL-NEXT: v_mul_hi_u32 v9, v14, v0
+; GISEL-NEXT: v_xor_b32_e32 v12, v2, v7
+; GISEL-NEXT: v_mul_lo_u32 v2, v16, v0
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v11
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_mul_hi_u32 v0, v16, v0
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v10, v2
+; GISEL-NEXT: v_mul_lo_u32 v10, v16, v11
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_mul_hi_u32 v10, v16, v11
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v9, v2
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v10, v2
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v15, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v8, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v11, v0
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v12, v1, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v11, 31, v3
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v11
-; GISEL-NEXT: v_xor_b32_e32 v12, v0, v11
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
-; GISEL-NEXT: v_mul_hi_u32 v13, v12, v7
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v3, v11, vcc
-; GISEL-NEXT: v_xor_b32_e32 v14, v2, v11
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v14, v7, 0
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v13, v0
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v8, 0
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v7, v2
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v13, v3, vcc
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v2, v0
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v8, v9, v0
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v7, 0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v10, v4
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v16, v2, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v12, v0
+; GISEL-NEXT: v_mul_lo_u32 v10, v12, v2
+; GISEL-NEXT: v_xor_b32_e32 v11, v3, v7
+; GISEL-NEXT: v_mul_lo_u32 v3, v11, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v11, v0
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v9, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v9, v12, v2
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v10, v3
+; GISEL-NEXT: v_mul_lo_u32 v10, v11, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
+; GISEL-NEXT: v_mul_hi_u32 v10, v11, v2
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v13, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
+; GISEL-NEXT: v_xor_b32_e32 v5, v5, v4
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v5, v4, vcc
-; GISEL-NEXT: v_mul_lo_u32 v4, v6, v7
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v8
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v14, v3
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v13, v[9:10]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v6, vcc
-; GISEL-NEXT: v_sub_i32_e64 v5, s[4:5], v2, v9
-; GISEL-NEXT: v_subbrev_u32_e64 v7, s[6:7], 0, v4, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[6:7]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v7, v6
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v6, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, v10, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], v5, v9
-; GISEL-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v14, v3, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v8
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v4, v5, v10, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v11
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v11
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v11, vcc
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v11, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v11, v3
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v8
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v9, vcc, v2, v6
+; GISEL-NEXT: v_subbrev_u32_e64 v10, s[4:5], 0, v3, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v6
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v10, v8
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
+; GISEL-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i64_pow2_shl_denom:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: s_mov_b64 s[4:5], 0x1000
-; CGP-NEXT: v_lshl_b64 v[11:12], s[4:5], v4
-; CGP-NEXT: v_mov_b32_e32 v5, v1
-; CGP-NEXT: v_mov_b32_e32 v7, v0
-; CGP-NEXT: v_lshl_b64 v[9:10], s[4:5], v6
-; CGP-NEXT: v_or_b32_e32 v0, v5, v12
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT: v_mov_b32_e32 v5, v2
+; CGP-NEXT: v_mov_b32_e32 v7, v3
+; CGP-NEXT: v_mov_b32_e32 v2, 0x1000
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_lshl_b64 v[11:12], v[2:3], v4
+; CGP-NEXT: v_mov_b32_e32 v9, v1
+; CGP-NEXT: v_mov_b32_e32 v8, v0
+; CGP-NEXT: v_or_b32_e32 v1, v9, v12
+; CGP-NEXT: v_mov_b32_e32 v0, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; CGP-NEXT: ; implicit-def: $vgpr0_vgpr1
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CGP-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
; CGP-NEXT: s_cbranch_execz .LBB8_2
; CGP-NEXT: ; %bb.1:
-; CGP-NEXT: v_ashrrev_i32_e32 v0, 31, v12
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v11, v0
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v12, v0, vcc
-; CGP-NEXT: v_xor_b32_e32 v6, v4, v0
-; CGP-NEXT: v_xor_b32_e32 v8, v1, v0
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v8
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v6
-; CGP-NEXT: v_sub_i32_e32 v14, vcc, 0, v8
-; CGP-NEXT: v_subb_u32_e32 v15, vcc, 0, v6, vcc
-; CGP-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CGP-NEXT: v_rcp_f32_e32 v0, v0
-; CGP-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; CGP-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; CGP-NEXT: v_trunc_f32_e32 v1, v1
-; CGP-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; CGP-NEXT: v_cvt_u32_f32_e32 v4, v1
-; CGP-NEXT: v_cvt_u32_f32_e32 v13, v0
-; CGP-NEXT: v_mul_lo_u32 v11, v14, v4
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v12, v15, v13
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v11
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v1, v12
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v13, v16, 0
-; CGP-NEXT: v_mul_hi_u32 v17, v13, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0
-; CGP-NEXT: v_add_i32_e32 v17, vcc, v17, v11
-; CGP-NEXT: v_addc_u32_e32 v18, vcc, 0, v12, vcc
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v16, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v17, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v18, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v12, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v11
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v0
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v4, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v13, 0
-; CGP-NEXT: v_mul_lo_u32 v11, v14, v4
-; CGP-NEXT: v_mul_lo_u32 v12, v15, v13
-; CGP-NEXT: v_mul_hi_u32 v15, v13, v0
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v11
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v1, v12
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v13, v14, 0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v15, v11
-; CGP-NEXT: v_addc_u32_e32 v16, vcc, 0, v12, vcc
-; CGP-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v14, 0
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v15, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v16, v1, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v12, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v11
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v0
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, v4, v1, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v13, 31, v5
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v7, v13
-; CGP-NEXT: v_xor_b32_e32 v7, v0, v13
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v12, 0
-; CGP-NEXT: v_mul_hi_u32 v14, v7, v11
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v5, v13, vcc
-; CGP-NEXT: v_xor_b32_e32 v15, v4, v13
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v15, v11, 0
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v14, v0
-; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v1, vcc
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v15, v12, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v11, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v14, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v0
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, 0, v1, vcc
-; CGP-NEXT: v_mul_lo_u32 v5, v8, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v4, v6, v4
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v5
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v15, v1
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v7, v0
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v6, vcc
-; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v0, v8
-; CGP-NEXT: v_subbrev_u32_e64 v7, s[6:7], 0, v4, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v7, v6
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v8
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[6:7], v7, v6
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v6, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[6:7]
-; CGP-NEXT: v_sub_i32_e64 v12, s[4:5], v5, v8
-; CGP-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v15, v1, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v11
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v6
-; CGP-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
-; CGP-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v4, v5, v12, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT: v_xor_b32_e32 v0, v0, v13
-; CGP-NEXT: v_xor_b32_e32 v1, v1, v13
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v13
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v13, vcc
+; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v12
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v11, v1
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, v12, v1, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v1
+; CGP-NEXT: v_xor_b32_e32 v1, v4, v1
+; CGP-NEXT: v_cvt_f32_u32_e32 v4, v0
+; CGP-NEXT: v_cvt_f32_u32_e32 v10, v1
+; CGP-NEXT: v_sub_i32_e32 v17, vcc, 0, v0
+; CGP-NEXT: v_subb_u32_e32 v18, vcc, 0, v1, vcc
+; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v10
+; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT: v_mul_f32_e32 v10, 0x2f800000, v4
+; CGP-NEXT: v_trunc_f32_e32 v10, v10
+; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v10
+; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT: v_cvt_u32_f32_e32 v16, v10
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v17, v4, 0
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[11:12]
+; CGP-NEXT: v_mul_hi_u32 v11, v4, v10
+; CGP-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v4, v[12:13]
+; CGP-NEXT: v_mul_lo_u32 v12, v16, v10
+; CGP-NEXT: v_mul_hi_u32 v10, v16, v10
+; CGP-NEXT: v_mul_lo_u32 v13, v4, v14
+; CGP-NEXT: v_mul_hi_u32 v15, v4, v14
+; CGP-NEXT: v_mul_lo_u32 v19, v16, v14
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v15
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v19
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_mul_hi_u32 v13, v16, v14
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT: v_addc_u32_e32 v16, vcc, v16, v11, vcc
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v17, v4, 0
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v17, v16, v[11:12]
+; CGP-NEXT: v_ashrrev_i32_e32 v17, 31, v9
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v17
+; CGP-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v18, v4, v[12:13]
+; CGP-NEXT: v_mul_hi_u32 v11, v4, v10
+; CGP-NEXT: v_xor_b32_e32 v15, v8, v17
+; CGP-NEXT: v_mul_lo_u32 v8, v16, v10
+; CGP-NEXT: v_mul_lo_u32 v12, v4, v14
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, v9, v17, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v16, v10
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v11, v4, v14
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v12, v8
+; CGP-NEXT: v_mul_lo_u32 v12, v16, v14
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT: v_mul_hi_u32 v12, v16, v14
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_addc_u32_e32 v8, vcc, v16, v10, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v15, v4
+; CGP-NEXT: v_mul_lo_u32 v11, v15, v8
+; CGP-NEXT: v_xor_b32_e32 v14, v9, v17
+; CGP-NEXT: v_mul_lo_u32 v9, v14, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v14, v4
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v15, v8
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_mul_lo_u32 v11, v14, v8
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT: v_mul_hi_u32 v12, v14, v8
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v4, 0
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v10
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v0, v12, v[9:10]
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v1, v4, v[10:11]
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v15, v8
+; CGP-NEXT: v_subb_u32_e64 v8, s[4:5], v14, v12, vcc
+; CGP-NEXT: v_sub_i32_e64 v9, s[4:5], v14, v12
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v1
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v0
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v8, v1
+; CGP-NEXT: v_subb_u32_e32 v9, vcc, v9, v1, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; CGP-NEXT: v_sub_i32_e32 v11, vcc, v4, v0
+; CGP-NEXT: v_subbrev_u32_e64 v12, s[4:5], 0, v9, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v1
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v0
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v12, v1
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v9, v1, vcc
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v11, v0
+; CGP-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[4:5]
+; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13
+; CGP-NEXT: v_cndmask_b32_e32 v0, v11, v0, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v12, v1, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v17
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v17
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v17
+; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v17, vcc
; CGP-NEXT: ; implicit-def: $vgpr11_vgpr12
-; CGP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CGP-NEXT: ; implicit-def: $vgpr8
; CGP-NEXT: .LBB8_2: ; %Flow1
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; CGP-NEXT: s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: v_lshl_b64 v[9:10], v[2:3], v6
+; CGP-NEXT: s_xor_b64 exec, exec, s[4:5]
; CGP-NEXT: s_cbranch_execz .LBB8_4
; CGP-NEXT: ; %bb.3:
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v11
@@ -1461,158 +2289,185 @@ define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; CGP-NEXT: v_mul_lo_u32 v1, v1, v0
; CGP-NEXT: v_mul_hi_u32 v1, v0, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_mul_hi_u32 v0, v7, v0
+; CGP-NEXT: v_mul_hi_u32 v0, v8, v0
+; CGP-NEXT: v_mov_b32_e32 v1, 0
; CGP-NEXT: v_mul_lo_u32 v0, v0, v11
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v7, v0
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v0, v11
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v8, v0
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v11
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v11
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v0, v11
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v11
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v11
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CGP-NEXT: v_mov_b32_e32 v1, 0
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; CGP-NEXT: .LBB8_4: ; %.split
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_or_b32_e32 v4, v3, v10
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT: v_or_b32_e32 v3, v7, v10
+; CGP-NEXT: v_mov_b32_e32 v2, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CGP-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; CGP-NEXT: s_cbranch_execz .LBB8_6
-; CGP-NEXT: ; %bb.5:
-; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v10
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v4
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v10, v4, vcc
-; CGP-NEXT: v_xor_b32_e32 v9, v6, v4
-; CGP-NEXT: v_xor_b32_e32 v10, v5, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v4, v10
-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v9
-; CGP-NEXT: v_sub_i32_e32 v13, vcc, 0, v10
-; CGP-NEXT: v_subb_u32_e32 v14, vcc, 0, v9, vcc
-; CGP-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CGP-NEXT: v_rcp_f32_e32 v4, v4
+; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
+; CGP-NEXT: s_cbranch_execnz .LBB8_7
+; CGP-NEXT: ; %bb.5: ; %Flow
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_cbranch_execnz .LBB8_8
+; CGP-NEXT: .LBB8_6: ; %.split.split
+; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
+; CGP-NEXT: s_setpc_b64 s[30:31]
+; CGP-NEXT: .LBB8_7:
+; CGP-NEXT: v_ashrrev_i32_e32 v3, 31, v10
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v9, v3
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, v10, v3, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v2, v3
+; CGP-NEXT: v_xor_b32_e32 v3, v4, v3
+; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2
+; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3
+; CGP-NEXT: v_sub_i32_e32 v14, vcc, 0, v2
+; CGP-NEXT: v_subb_u32_e32 v15, vcc, 0, v3, vcc
+; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v6
+; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CGP-NEXT: v_trunc_f32_e32 v5, v5
-; CGP-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CGP-NEXT: v_cvt_u32_f32_e32 v11, v5
-; CGP-NEXT: v_cvt_u32_f32_e32 v12, v4
-; CGP-NEXT: v_mul_lo_u32 v6, v13, v11
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; CGP-NEXT: v_mul_lo_u32 v7, v14, v12
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v15, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v15, 0
-; CGP-NEXT: v_mul_hi_u32 v16, v12, v4
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; CGP-NEXT: v_add_i32_e32 v16, vcc, v16, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v11, v15, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v16, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v4
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, v11, v5, vcc
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; CGP-NEXT: v_mul_lo_u32 v6, v13, v11
-; CGP-NEXT: v_mul_lo_u32 v7, v14, v12
-; CGP-NEXT: v_mul_hi_u32 v14, v12, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v13, 0
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v5
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v11, v13, 0
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v6, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v14, v7
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v12, v4
-; CGP-NEXT: v_addc_u32_e32 v6, vcc, v11, v5, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v3, v7, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v6, 0
-; CGP-NEXT: v_mul_hi_u32 v11, v8, v4
-; CGP-NEXT: v_xor_b32_e32 v12, v5, v7
-; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v4, 0
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v2
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v3, vcc
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v6, 0
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v11, v4
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v13, v5, vcc
-; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v2
-; CGP-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; CGP-NEXT: v_mul_lo_u32 v5, v10, v2
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v4, 0
-; CGP-NEXT: v_mul_lo_u32 v4, v9, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v12, v3
+; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT: v_trunc_f32_e32 v6, v6
+; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
+; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v14, v4, 0
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v6, v[9:10]
+; CGP-NEXT: v_mul_hi_u32 v9, v4, v8
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v15, v4, v[10:11]
+; CGP-NEXT: v_mul_lo_u32 v10, v6, v8
+; CGP-NEXT: v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT: v_mul_lo_u32 v11, v4, v12
+; CGP-NEXT: v_mul_hi_u32 v13, v4, v12
+; CGP-NEXT: v_mul_lo_u32 v16, v6, v12
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v13
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v16
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_mul_hi_u32 v11, v6, v12
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, v6, v9, vcc
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v14, v4, 0
+; CGP-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v6, v[9:10]
+; CGP-NEXT: v_ashrrev_i32_e32 v14, 31, v7
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v14
+; CGP-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v15, v4, v[10:11]
+; CGP-NEXT: v_mul_hi_u32 v9, v4, v8
+; CGP-NEXT: v_xor_b32_e32 v11, v5, v14
+; CGP-NEXT: v_mul_lo_u32 v5, v6, v8
+; CGP-NEXT: v_mul_lo_u32 v10, v4, v12
+; CGP-NEXT: v_addc_u32_e32 v7, vcc, v7, v14, vcc
+; CGP-NEXT: v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v9, v4, v12
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT: v_mul_lo_u32 v10, v6, v12
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_mul_hi_u32 v10, v6, v12
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_addc_u32_e32 v5, vcc, v6, v8, vcc
+; CGP-NEXT: v_mul_hi_u32 v6, v11, v4
+; CGP-NEXT: v_mul_lo_u32 v8, v11, v5
+; CGP-NEXT: v_xor_b32_e32 v10, v7, v14
+; CGP-NEXT: v_mul_lo_u32 v7, v10, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v10, v4
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v7, v11, v5
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT: v_mul_lo_u32 v8, v10, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v4, v6
+; CGP-NEXT: v_mul_hi_u32 v8, v10, v5
+; CGP-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v12, 0
+; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v6
+; CGP-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v2, v8, v[5:6]
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v11, v4
+; CGP-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v3, v12, v[6:7]
+; CGP-NEXT: v_subb_u32_e64 v5, s[4:5], v10, v8, vcc
+; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v10, v8
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v3
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v2
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v5, v3
+; CGP-NEXT: v_subb_u32_e32 v6, vcc, v6, v3, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[4:5]
+; CGP-NEXT: v_sub_i32_e32 v8, vcc, v4, v2
+; CGP-NEXT: v_subbrev_u32_e64 v9, s[4:5], 0, v6, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v3
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v2
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v3
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v6, v3, vcc
; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v9, vcc
-; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v2, v10
-; CGP-NEXT: v_subbrev_u32_e64 v6, s[6:7], 0, v4, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v9
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v10
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[6:7], v6, v9
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v9, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[6:7]
-; CGP-NEXT: v_sub_i32_e64 v11, s[4:5], v5, v10
-; CGP-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CGP-NEXT: v_subb_u32_e32 v3, vcc, v12, v3, vcc
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v8
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v9
-; CGP-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v10
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v9
-; CGP-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v4, v5, v11, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; CGP-NEXT: v_xor_b32_e32 v2, v2, v7
-; CGP-NEXT: v_xor_b32_e32 v3, v3, v7
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v7
-; CGP-NEXT: v_subb_u32_e32 v5, vcc, v3, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; CGP-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; CGP-NEXT: v_xor_b32_e32 v2, v2, v14
+; CGP-NEXT: v_xor_b32_e32 v3, v3, v14
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v14
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v14, vcc
; CGP-NEXT: ; implicit-def: $vgpr9_vgpr10
-; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CGP-NEXT: .LBB8_6: ; %Flow
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; CGP-NEXT: s_cbranch_execz .LBB8_8
-; CGP-NEXT: ; %bb.7:
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v9
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v9
-; CGP-NEXT: v_mov_b32_e32 v5, 0
-; CGP-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v4, v3
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: ; implicit-def: $vgpr5
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_cbranch_execz .LBB8_6
+; CGP-NEXT: .LBB8_8:
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v9
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v9
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v2
+; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
-; CGP-NEXT: v_mul_lo_u32 v3, v3, v9
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v9
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v2, v5, v2
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_mul_lo_u32 v2, v2, v9
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v9
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v9
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v9
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v9
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v9
-; CGP-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc
-; CGP-NEXT: .LBB8_8: ; %.split.split
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_mov_b32_e32 v2, v4
-; CGP-NEXT: v_mov_b32_e32 v3, v5
; CGP-NEXT: s_setpc_b64 s[30:31]
%shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
%r = srem <2 x i64> %x, %shl.y
@@ -1887,45 +2742,45 @@ define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; CGP-LABEL: v_srem_v2i64_24bit:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v1
-; CGP-NEXT: v_and_b32_e32 v5, 0xffffff, v6
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, 0, v1
-; CGP-NEXT: v_rcp_f32_e32 v3, v3
-; CGP-NEXT: v_cvt_f32_u32_e32 v7, v5
-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
+; CGP-NEXT: v_and_b32_e32 v5, 0xffffff, v4
+; CGP-NEXT: v_cvt_f32_u32_e32 v1, v5
+; CGP-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; CGP-NEXT: v_cvt_f32_u32_e32 v3, v6
+; CGP-NEXT: v_and_b32_e32 v7, 0xffffff, v0
+; CGP-NEXT: v_rcp_f32_e32 v1, v1
; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_rcp_f32_e32 v7, v7
-; CGP-NEXT: v_mul_lo_u32 v8, v8, v3
-; CGP-NEXT: v_mul_f32_e32 v7, 0x4f7ffffe, v7
-; CGP-NEXT: v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT: v_mul_hi_u32 v8, v3, v8
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v8
-; CGP-NEXT: v_mul_hi_u32 v3, v3, v0
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT: v_mul_lo_u32 v8, v8, v7
-; CGP-NEXT: v_mul_u32_u24_e32 v3, v3, v4
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v0, v1
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; CGP-NEXT: v_cvt_u32_f32_e32 v4, v1
+; CGP-NEXT: v_rcp_f32_e32 v1, v3
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v5
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v4
+; CGP-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1
+; CGP-NEXT: v_cvt_u32_f32_e32 v8, v0
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v3, 0
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, 0, v6
+; CGP-NEXT: v_mul_lo_u32 v9, v0, v8
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v4, v1
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v3, 0
+; CGP-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v9, 0
+; CGP-NEXT: v_mul_lo_u32 v0, v1, v5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v8, v4
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v7, v0
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v2, v3, 0
+; CGP-NEXT: v_sub_i32_e32 v7, vcc, v4, v5
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc
+; CGP-NEXT: v_mul_lo_u32 v4, v1, v6
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v0, v5
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; CGP-NEXT: v_mul_hi_u32 v3, v7, v8
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v1
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v7, v3
-; CGP-NEXT: v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; CGP-NEXT: v_mul_u32_u24_e32 v3, v3, v6
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v5
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v5
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; CGP-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; CGP-NEXT: s_setpc_b64 s[30:31]
%num.mask = and <2 x i64> %num, <i64 16777215, i64 16777215>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index 1973318a15513..4d34ce50bc98f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -93,17 +93,22 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
;
; GFX8-LABEL: s_ssubsat_i7:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b32 s1, s1, 9
; GFX8-NEXT: s_lshl_b32 s0, s0, 9
+; GFX8-NEXT: s_sext_i32_i16 s2, s0
+; GFX8-NEXT: s_max_i32 s3, s2, -1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 9
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8001
+; GFX8-NEXT: s_min_i32 s2, s2, -1
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 9
-; GFX8-NEXT: s_lshr_b32 s0, s0, 9
+; GFX8-NEXT: s_add_i32 s2, s2, 0x8000
+; GFX8-NEXT: s_max_i32 s1, s3, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_min_i32 s1, s1, s2
; GFX8-NEXT: s_sub_i32 s0, s0, s1
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_not_b32_e32 v0, 63
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, 63
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s0, s0, 9
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_i7:
@@ -112,39 +117,39 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
; GFX9-NEXT: s_lshl_b32 s0, s0, 9
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_sub_i16 v0, s0, v0 clamp
-; GFX9-NEXT: v_mov_b32_e32 v1, 9
-; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 9
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_ssubsat_i7:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshl_b32 s1, s1, 9
; GFX10-NEXT: s_lshl_b32 s0, s0, 9
-; GFX10-NEXT: v_mov_b32_e32 v1, 9
+; GFX10-NEXT: s_lshl_b32 s1, s1, 9
; GFX10-NEXT: v_sub_nc_i16 v0, s0, s1 clamp
-; GFX10-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_sext_i32_i16 s0, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 9
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_ssubsat_i7:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-TRUE16-NEXT: v_sub_nc_i16 v0.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v0, 9, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 9
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_ssubsat_i7:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 9
; GFX11-FAKE16-NEXT: v_sub_nc_i16 v0, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v0, 9, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 9
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i7 @llvm.ssub.sat.i7(i7 %lhs, i7 %rhs)
ret i7 %result
@@ -237,14 +242,22 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
;
; GFX8-LABEL: s_ssubsat_i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x80000
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s2, s0
+; GFX8-NEXT: s_max_i32 s3, s2, -1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8001
+; GFX8-NEXT: s_min_i32 s2, s2, -1
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_add_i32 s2, s2, 0x8000
+; GFX8-NEXT: s_max_i32 s1, s3, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_min_i32 s1, s1, s2
; GFX8-NEXT: s_sub_i32 s0, s0, s1
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7f
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s0, s0, 8
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_i8:
@@ -253,39 +266,39 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshl_b32 s0, s0, 8
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_sub_i16 v0, s0, v0 clamp
-; GFX9-NEXT: v_mov_b32_e32 v1, 8
-; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 8
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_ssubsat_i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_lshl_b32 s0, s0, 8
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: v_sub_nc_i16 v0, s0, s1 clamp
-; GFX10-NEXT: v_ashrrev_i32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_sext_i32_i16 s0, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 8
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_ssubsat_i8:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-TRUE16-NEXT: v_sub_nc_i16 v0.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_ssubsat_i8:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-FAKE16-NEXT: v_sub_nc_i16 v0, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v0, 8, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 8
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i8 @llvm.ssub.sat.i8(i8 %lhs, i8 %rhs)
ret i8 %result
@@ -450,23 +463,44 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX8-LABEL: s_ssubsat_v2i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_sext_i32_i16 s2, s1
-; GFX8-NEXT: s_sext_i32_i16 s3, s0
-; GFX8-NEXT: s_ashr_i32 s2, s2, 8
-; GFX8-NEXT: s_ashr_i32 s3, s3, 8
-; GFX8-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x80000
-; GFX8-NEXT: s_sub_i32 s2, s3, s2
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s4, s0
+; GFX8-NEXT: s_max_i32 s5, s4, -1
+; GFX8-NEXT: s_lshr_b32 s3, s1, 8
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_add_i32 s5, s5, 0x8001
+; GFX8-NEXT: s_min_i32 s4, s4, -1
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8000
+; GFX8-NEXT: s_max_i32 s1, s5, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_min_i32 s1, s1, s4
; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: v_med3_i32 v2, s2, v0, v1
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: s_sext_i32_i16 s3, s1
+; GFX8-NEXT: s_max_i32 s4, s3, -1
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8001
+; GFX8-NEXT: s_min_i32 s3, s3, -1
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8000
+; GFX8-NEXT: s_max_i32 s2, s4, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_min_i32 s2, s2, s3
+; GFX8-NEXT: s_sub_i32 s1, s1, s2
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s1, s1, 8
+; GFX8-NEXT: s_ashr_i32 s0, s0, 8
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_v2i8:
@@ -498,14 +532,31 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX11-TRUE16-LABEL: s_ssubsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s1, 8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s0, 8
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v0.l, s3, s2 clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v1.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0105
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, s0, s1 clamp
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s1, 0x80008
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s2, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s2, s1
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_ssubsat_v2i8:
@@ -2505,20 +2556,18 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
;
; GFX8-LABEL: s_ssubsat_i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_sub_i32 s4, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s4, 16
+; GFX8-NEXT: s_sext_i32_i16 s2, s0
+; GFX8-NEXT: s_max_i32 s3, s2, -1
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8001
+; GFX8-NEXT: s_min_i32 s2, s2, -1
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s0, s1
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s2, 0
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: s_sext_i32_i16 s2, s4
-; GFX8-NEXT: s_ashr_i32 s2, s2, 15
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX8-NEXT: s_cselect_b32 s0, s2, s4
+; GFX8-NEXT: s_add_i32 s2, s2, 0x8000
+; GFX8-NEXT: s_max_i32 s1, s3, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_min_i32 s1, s1, s2
+; GFX8-NEXT: s_sub_i32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_i16:
@@ -3784,110 +3833,87 @@ define i48 @v_ssubsat_i48(i48 %lhs, i48 %rhs) {
; GFX6-LABEL: v_ssubsat_i48:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 16
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 16
-; GFX6-NEXT: v_ashr_i64 v[2:3], v[2:3], 16
-; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_mov_b32 s4, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT: s_movk_i32 s5, 0x7fff
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: s_mov_b32 s4, 0
-; GFX6-NEXT: s_movk_i32 s5, 0x8000
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT: v_subb_u32_e32 v6, vcc, v1, v3, vcc
+; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT: v_bfe_i32 v5, v6, 0, 16
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[2:3]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xffff8000, v0
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 15, v5
+; GFX6-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_i48:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
-; GFX8-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_mov_b32 s4, -1
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8-NEXT: s_movk_i32 s5, 0x7fff
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX8-NEXT: s_mov_b32 s4, 0
-; GFX8-NEXT: s_movk_i32 s5, 0x8000
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v1, v3, vcc
+; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX8-NEXT: v_bfe_i32 v5, v6, 0, 16
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0xffff8000, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 15, v5
+; GFX8-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_i48:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
+; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
+; GFX9-NEXT: s_xor_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: s_mov_b32 s4, -1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9-NEXT: s_movk_i32 s5, 0x7fff
-; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX9-NEXT: s_mov_b32 s4, 0
-; GFX9-NEXT: s_movk_i32 s5, 0x8000
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_ssubsat_i48:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: s_mov_b32 s4, -1
-; GFX10-NEXT: s_movk_i32 s5, 0x7fff
-; GFX10-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
+; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v0, v2
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX10-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX10-NEXT: v_cmp_gt_i64_e64 s4, 0, v[4:5]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
+; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: s_movk_i32 s5, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fff, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff8000, v1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_ssubsat_i48:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, -1
-; GFX11-NEXT: s_movk_i32 s1, 0x7fff
-; GFX11-NEXT: v_ashrrev_i64 v[2:3], 16, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 16, v[2:3]
+; GFX11-NEXT: v_sub_co_u32 v4, vcc_lo, v0, v2
+; GFX11-NEXT: v_sub_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX11-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[4:5]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
+; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v6 :: v_dual_cndmask_b32 v1, v5, v1
; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_movk_i32 s1, 0x8000
-; GFX11-NEXT: v_dual_cndmask_b32 v1, 0x7fff, v1 :: v_dual_cndmask_b32 v0, -1, v0
-; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_dual_cndmask_b32 v0, 0, v0 :: v_dual_cndmask_b32 v1, 0xffff8000, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i48 @llvm.ssub.sat.i48(i48 %lhs, i48 %rhs)
ret i48 %result
@@ -3988,49 +4014,39 @@ define amdgpu_ps i48 @s_ssubsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
define amdgpu_ps <2 x float> @ssubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX6-LABEL: ssubsat_i48_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GFX6-NEXT: s_mov_b32 s0, -1
-; GFX6-NEXT: s_movk_i32 s1, 0x7fff
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: s_mov_b32 s0, 0
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v3, v1, vcc
+; GFX6-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xffff8000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i48_sv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GFX8-NEXT: s_mov_b32 s0, -1
-; GFX8-NEXT: s_movk_i32 s1, 0x7fff
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX8-NEXT: s_mov_b32 s0, 0
-; GFX8-NEXT: s_movk_i32 s1, 0x8000
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT: v_subb_u32_e32 v4, vcc, v3, v1, vcc
+; GFX8-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xffff8000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: ; return to shader part epilog
;
@@ -4038,23 +4054,17 @@ define amdgpu_ps <2 x float> @ssubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
-; GFX9-NEXT: s_mov_b32 s0, -1
-; GFX9-NEXT: s_movk_i32 s1, 0x7fff
-; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX9-NEXT: s_mov_b32 s0, 0
-; GFX9-NEXT: s_movk_i32 s1, 0x8000
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v1, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[0:1], 0, v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
+; GFX9-NEXT: s_xor_b64 vcc, vcc, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -4062,20 +4072,16 @@ define amdgpu_ps <2 x float> @ssubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX10-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, s0, v0
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s0, -1
-; GFX10-NEXT: s_movk_i32 s1, 0x7fff
-; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_movk_i32 s1, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fff, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, s0, v0
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff8000, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: ; return to shader part epilog
;
@@ -4083,18 +4089,15 @@ define amdgpu_ps <2 x float> @ssubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX11: ; %bb.0:
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX11-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, s0, v0
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX11-NEXT: s_mov_b32 s0, -1
-; GFX11-NEXT: s_movk_i32 s1, 0x7fff
-; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_movk_i32 s1, 0x8000
-; GFX11-NEXT: v_dual_cndmask_b32 v1, 0x7fff, v1 :: v_dual_cndmask_b32 v0, -1, v0
+; GFX11-NEXT: v_sub_co_u32 v2, vcc_lo, s0, v0
+; GFX11-NEXT: v_sub_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_dual_cndmask_b32 v0, 0, v0 :: v_dual_cndmask_b32 v1, 0xffff8000, v1
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
+; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i48 @llvm.ssub.sat.i48(i48 %lhs, i48 %rhs)
@@ -4106,49 +4109,39 @@ define amdgpu_ps <2 x float> @ssubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
define amdgpu_ps <2 x float> @ssubsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX6-LABEL: ssubsat_i48_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 16
-; GFX6-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GFX6-NEXT: s_mov_b32 s0, -1
-; GFX6-NEXT: s_movk_i32 s1, 0x7fff
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: s_mov_b32 s0, 0
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v1, v3, vcc
+; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX6-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xffff8000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i48_vs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX8-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GFX8-NEXT: s_mov_b32 s0, -1
-; GFX8-NEXT: s_movk_i32 s1, 0x7fff
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX8-NEXT: s_mov_b32 s0, 0
-; GFX8-NEXT: s_movk_i32 s1, 0x8000
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT: v_subb_u32_e32 v4, vcc, v1, v3, vcc
+; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x300000
+; GFX8-NEXT: v_bfe_i32 v3, v4, 0, 16
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 15, v3
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xffff8000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: ; return to shader part epilog
;
@@ -4156,23 +4149,17 @@ define amdgpu_ps <2 x float> @ssubsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX9-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v2, vcc
-; GFX9-NEXT: s_mov_b32 s0, -1
-; GFX9-NEXT: s_movk_i32 s1, 0x7fff
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_subrev_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX9-NEXT: s_mov_b32 s0, 0
-; GFX9-NEXT: s_movk_i32 s1, 0x8000
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff8000
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[0:1], 0, v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
+; GFX9-NEXT: s_xor_b64 vcc, vcc, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: ; return to shader part epilog
;
@@ -4180,20 +4167,16 @@ define amdgpu_ps <2 x float> @ssubsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX10-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s0
-; GFX10-NEXT: v_subrev_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s0, -1
-; GFX10-NEXT: s_movk_i32 s1, 0x7fff
+; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s0
+; GFX10-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_movk_i32 s1, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fff, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff8000, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: ; return to shader part epilog
;
@@ -4201,18 +4184,15 @@ define amdgpu_ps <2 x float> @ssubsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX11: ; %bb.0:
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
-; GFX11-NEXT: s_ashr_i64 s[0:1], s[0:1], 16
-; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s0
-; GFX11-NEXT: v_subrev_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX11-NEXT: s_mov_b32 s0, -1
-; GFX11-NEXT: s_movk_i32 s1, 0x7fff
+; GFX11-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s0
+; GFX11-NEXT: v_subrev_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_movk_i32 s1, 0x8000
-; GFX11-NEXT: v_dual_cndmask_b32 v1, 0x7fff, v1 :: v_dual_cndmask_b32 v0, -1, v0
-; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_dual_cndmask_b32 v0, 0, v0 :: v_dual_cndmask_b32 v1, 0xffff8000, v1
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
+; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
+; GFX11-NEXT: v_ashrrev_i64 v[0:1], 16, v[0:1]
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i48 @llvm.ssub.sat.i48(i48 %lhs, i48 %rhs)
@@ -4225,40 +4205,43 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
; GFX6-LABEL: v_ssubsat_i64:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[2:3]
-; GFX6-NEXT: v_sub_i32_e64 v0, s[4:5], v0, v2
-; GFX6-NEXT: v_subb_u32_e64 v1, s[4:5], v1, v3, s[4:5]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX6-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT: v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[2:3]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_i64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: v_sub_u32_e64 v0, s[4:5], v0, v2
-; GFX8-NEXT: v_subb_u32_e64 v1, s[4:5], v1, v3, s[4:5]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x80000000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v3, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT: v_sub_co_u32_e64 v0, s[4:5], v0, v2
-; GFX9-NEXT: v_subb_co_u32_e64 v1, s[4:5], v1, v3, s[4:5]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_ssubsat_i64:
@@ -4267,11 +4250,12 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v0, v2
; GFX10-NEXT: v_sub_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX10-NEXT: v_cmp_gt_i32_e64 s4, 0, v5
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v5
+; GFX10-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX10-NEXT: v_cmp_gt_i64_e64 s4, 0, v[4:5]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_ssubsat_i64:
@@ -4280,11 +4264,11 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
; GFX11-NEXT: v_sub_co_u32 v4, vcc_lo, v0, v2
; GFX11-NEXT: v_sub_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v5
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v5
+; GFX11-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[4:5]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v6
; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v6 :: v_dual_cndmask_b32 v1, v5, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
ret i64 %result
@@ -4371,41 +4355,44 @@ define amdgpu_ps i64 @s_ssubsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
define amdgpu_ps <2 x float> @ssubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
; GFX6-LABEL: ssubsat_i64_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_sub_i32_e64 v0, s[0:1], s0, v0
-; GFX6-NEXT: v_subb_u32_e64 v1, s[0:1], v2, v1, s[0:1]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX6-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v3, v1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i64_sv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_sub_u32_e64 v0, s[0:1], s0, v0
-; GFX8-NEXT: v_subb_u32_e64 v1, s[0:1], v2, v1, s[0:1]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT: v_subb_u32_e32 v3, vcc, v3, v1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x80000000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i64_sv:
; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v1, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_sub_co_u32_e64 v0, s[0:1], s0, v0
-; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v2, v1, s[0:1]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[0:1], 0, v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: ssubsat_i64_sv:
@@ -4413,11 +4400,12 @@ define amdgpu_ps <2 x float> @ssubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, s0, v0
; GFX10-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: v_cmp_gt_i32_e64 s0, 0, v3
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: ssubsat_i64_sv:
@@ -4425,11 +4413,11 @@ define amdgpu_ps <2 x float> @ssubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
; GFX11-NEXT: v_sub_co_u32 v2, vcc_lo, s0, v0
; GFX11-NEXT: v_sub_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v3
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
%cast = bitcast i64 %result to <2 x float>
@@ -4439,41 +4427,44 @@ define amdgpu_ps <2 x float> @ssubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
define amdgpu_ps <2 x float> @ssubsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
; GFX6-LABEL: ssubsat_i64_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_subrev_i32_e64 v0, s[0:1], s0, v0
-; GFX6-NEXT: v_subb_u32_e64 v1, s[0:1], v1, v2, s[0:1]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX6-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX6-NEXT: v_mov_b32_e32 v3, s1
+; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000000, v0
+; GFX6-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i64_vs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_subrev_u32_e64 v0, s[0:1], s0, v0
-; GFX8-NEXT: v_subb_u32_e64 v1, s[0:1], v1, v2, s[0:1]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[0:1], s[0:1], v[0:1]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[2:3], 0, v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x80000000, v0
+; GFX8-NEXT: s_xor_b64 vcc, s[0:1], s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i64_vs:
; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_subrev_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[0:1], s0, v0
-; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v1, v2, s[0:1]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[0:1], 0, v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: ssubsat_i64_vs:
@@ -4481,11 +4472,12 @@ define amdgpu_ps <2 x float> @ssubsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s0
; GFX10-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT: v_cmp_gt_i32_e64 s0, 0, v3
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: ssubsat_i64_vs:
@@ -4493,11 +4485,11 @@ define amdgpu_ps <2 x float> @ssubsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
; GFX11-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s0
; GFX11-NEXT: v_subrev_co_ci_u32_e64 v3, null, s1, v1, vcc_lo
; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v3
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v3
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v4
; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v3, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
%cast = bitcast i64 %result to <2 x float>
@@ -4508,64 +4500,72 @@ define <2 x i64> @v_ssubsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX6-LABEL: v_ssubsat_v2i64:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]
-; GFX6-NEXT: v_sub_i32_e64 v0, s[4:5], v0, v4
-; GFX6-NEXT: v_subb_u32_e64 v1, s[4:5], v1, v5, s[4:5]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; GFX6-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, -v4, vcc
-; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[6:7]
-; GFX6-NEXT: v_sub_i32_e64 v2, s[4:5], v2, v6
-; GFX6-NEXT: v_subb_u32_e64 v3, s[4:5], v3, v7, s[4:5]
-; GFX6-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v3
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX6-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, -v4, vcc
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, v0, v4
+; GFX6-NEXT: v_subb_u32_e32 v9, vcc, v1, v5, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[4:5]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[8:9]
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v9
+; GFX6-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v0, v1
+; GFX6-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v2, v6
+; GFX6-NEXT: v_subb_u32_e32 v5, vcc, v3, v7, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e64 s[4:5], v[2:3], v[6:7]
+; GFX6-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0x80000000, v2
+; GFX6-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_v2i64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]
-; GFX8-NEXT: v_sub_u32_e64 v0, s[4:5], v0, v4
-; GFX8-NEXT: v_subb_u32_e64 v1, s[4:5], v1, v5, s[4:5]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, -v4, vcc
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[6:7]
-; GFX8-NEXT: v_sub_u32_e64 v2, s[4:5], v2, v6
-; GFX8-NEXT: v_subb_u32_e64 v3, s[4:5], v3, v7, s[4:5]
-; GFX8-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, -v4, vcc
+; GFX8-NEXT: v_sub_u32_e32 v8, vcc, v0, v4
+; GFX8-NEXT: v_subb_u32_e32 v9, vcc, v1, v5, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[4:5]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[8:9]
+; GFX8-NEXT: v_ashrrev_i32_e32 v0, 31, v9
+; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v2, v6
+; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v3, v7, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e64 s[4:5], v[2:3], v[6:7]
+; GFX8-NEXT: v_cmp_gt_i64_e64 s[6:7], 0, v[4:5]
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x80000000, v2
+; GFX8-NEXT: s_xor_b64 vcc, s[4:5], s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v0, v4
+; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v1, v5, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]
-; GFX9-NEXT: v_sub_co_u32_e64 v0, s[4:5], v0, v4
-; GFX9-NEXT: v_subb_co_u32_e64 v1, s[4:5], v1, v5, s[4:5]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[8:9]
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v9
+; GFX9-NEXT: v_add_u32_e32 v1, 0x80000000, v0
; GFX9-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v2, v6
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v7, vcc
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[6:7]
-; GFX9-NEXT: v_sub_co_u32_e64 v2, s[4:5], v2, v6
-; GFX9-NEXT: v_subb_co_u32_e64 v3, s[4:5], v3, v7, s[4:5]
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v3
-; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX9-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX9-NEXT: v_add_u32_e32 v3, 0x80000000, v2
; GFX9-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, -v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_ssubsat_v2i64:
@@ -4575,18 +4575,20 @@ define <2 x i64> @v_ssubsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX10-NEXT: v_sub_co_ci_u32_e32 v9, vcc_lo, v1, v5, vcc_lo
; GFX10-NEXT: v_sub_co_u32 v10, vcc_lo, v2, v6
; GFX10-NEXT: v_sub_co_ci_u32_e32 v11, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT: v_ashrrev_i32_e32 v12, 31, v9
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX10-NEXT: v_cmp_gt_i32_e64 s4, 0, v9
+; GFX10-NEXT: v_cmp_gt_i64_e64 s4, 0, v[8:9]
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v11
; GFX10-NEXT: v_cmp_lt_i64_e64 s5, v[2:3], v[6:7]
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v9
-; GFX10-NEXT: v_cmp_gt_i32_e64 s6, 0, v11
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v11
+; GFX10-NEXT: v_cmp_gt_i64_e64 s6, 0, v[10:11]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v12
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v4
; GFX10-NEXT: s_xor_b32 vcc_lo, vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, -v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v12, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo
; GFX10-NEXT: s_xor_b32 vcc_lo, s5, s6
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, -v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v10, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_ssubsat_v2i64:
@@ -4596,18 +4598,18 @@ define <2 x i64> @v_ssubsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
; GFX11-NEXT: v_sub_co_ci_u32_e64 v9, null, v1, v5, vcc_lo
; GFX11-NEXT: v_sub_co_u32 v10, vcc_lo, v2, v6
; GFX11-NEXT: v_sub_co_ci_u32_e64 v11, null, v3, v7, vcc_lo
+; GFX11-NEXT: v_ashrrev_i32_e32 v12, 31, v9
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v9
+; GFX11-NEXT: v_cmp_gt_i64_e64 s0, 0, v[8:9]
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v11
; GFX11-NEXT: v_cmp_lt_i64_e64 s1, v[2:3], v[6:7]
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v9
-; GFX11-NEXT: v_cmp_gt_i32_e64 s2, 0, v11
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v11
+; GFX11-NEXT: v_cmp_gt_i64_e64 s2, 0, v[10:11]
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x80000000, v12
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v4
; GFX11-NEXT: s_xor_b32 vcc_lo, vcc_lo, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, -v1, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v8, v12 :: v_dual_cndmask_b32 v1, v9, v1
; GFX11-NEXT: s_xor_b32 vcc_lo, s1, s2
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, -v3, vcc_lo
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v10, v4 :: v_dual_cndmask_b32 v3, v11, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
ret <2 x i64> %result
@@ -4891,96 +4893,165 @@ define amdgpu_ps <4 x float> @ssubsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
; GFX6-LABEL: ssubsat_i128_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_mov_b32_e32 v4, s1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v4, v1, vcc
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s0, v0
+; GFX6-NEXT: v_subb_u32_e32 v7, vcc, v4, v1, vcc
; GFX6-NEXT: v_mov_b32_e32 v4, s2
; GFX6-NEXT: v_mov_b32_e32 v5, s3
-; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v4, v2, vcc
-; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v5, v3, vcc
-; GFX6-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX6-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX6-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v4, v2, vcc
+; GFX6-NEXT: v_subb_u32_e32 v5, vcc, v5, v3, vcc
+; GFX6-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX6-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0x80000000, v2
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i128_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v4, s1
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v4, v1, vcc
+; GFX8-NEXT: v_sub_u32_e32 v6, vcc, s0, v0
+; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v4, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: v_mov_b32_e32 v5, s3
-; GFX8-NEXT: v_subb_u32_e32 v2, vcc, v4, v2, vcc
-; GFX8-NEXT: v_subb_u32_e32 v4, vcc, v5, v3, vcc
-; GFX8-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX8-NEXT: v_subb_u32_e32 v4, vcc, v4, v2, vcc
+; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v5, v3, vcc
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x80000000, v2
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i128_sv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v4, v1, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, s0, v0
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v4, v1, vcc
; GFX9-NEXT: v_mov_b32_e32 v4, s2
; GFX9-NEXT: v_mov_b32_e32 v5, s3
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v2, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v5, v3, vcc
-; GFX9-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v2, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v3, vcc
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX9-NEXT: v_add_u32_e32 v3, 0x80000000, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: ssubsat_i128_sv:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, s0, v0
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v4, vcc_lo, s3, v3, vcc_lo
-; GFX10-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX10-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
+; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, s0, v0
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v7, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v5, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[4:5]
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[4:5]
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v2
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v8, 0, vcc_lo
+; GFX10-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: ssubsat_i128_sv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, s0, v0
-; GFX11-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX11-NEXT: v_sub_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v4, null, s3, v3, vcc_lo
-; GFX11-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX11-NEXT: v_xor_b32_e32 v5, s3, v4
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: ssubsat_i128_sv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_sub_co_u32 v6, vcc_lo, s0, v0
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e32 v7, vcc_lo, s1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e64 v5, null, s3, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmp_lt_i64_e64 s0, s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s2, s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e64 s1, 0, v[4:5]
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[4:5]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, 0, vcc_lo
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v6, v2 :: v_dual_cndmask_b32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: ssubsat_i128_sv:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_sub_co_u32 v6, vcc_lo, s0, v0
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e32 v7, vcc_lo, s1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e64 v5, null, s3, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[4:5]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v1, v0 :: v_dual_add_nc_u32 v3, 0x80000000, v2
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[4:5]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v8, 0, vcc_lo
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v6, v2 :: v_dual_cndmask_b32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i128 @llvm.ssub.sat.i128(i128 %lhs, i128 %rhs)
%cast = bitcast i128 %result to <4 x float>
ret <4 x float> %cast
@@ -4990,96 +5061,165 @@ define amdgpu_ps <4 x float> @ssubsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
; GFX6-LABEL: ssubsat_i128_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_mov_b32_e32 v4, s1
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GFX6-NEXT: v_subrev_i32_e32 v6, vcc, s0, v0
+; GFX6-NEXT: v_subb_u32_e32 v7, vcc, v1, v4, vcc
; GFX6-NEXT: v_mov_b32_e32 v4, s2
; GFX6-NEXT: v_mov_b32_e32 v5, s3
-; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v2, v4, vcc
-; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v3, v5, vcc
-; GFX6-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX6-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX6-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v2, v4, vcc
+; GFX6-NEXT: v_subb_u32_e32 v5, vcc, v3, v5, vcc
+; GFX6-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX6-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0x80000000, v2
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i128_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v4, s1
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s0, v0
+; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v1, v4, vcc
; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: v_mov_b32_e32 v5, s3
-; GFX8-NEXT: v_subb_u32_e32 v2, vcc, v2, v4, vcc
-; GFX8-NEXT: v_subb_u32_e32 v4, vcc, v3, v5, vcc
-; GFX8-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX8-NEXT: v_subb_u32_e32 v4, vcc, v2, v4, vcc
+; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v3, v5, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x80000000, v2
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i128_vs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v4, vcc
+; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s0, v0
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v1, v4, vcc
; GFX9-NEXT: v_mov_b32_e32 v4, s2
; GFX9-NEXT: v_mov_b32_e32 v5, s3
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v3, v5, vcc
-; GFX9-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v4, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v5, vcc
+; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX9-NEXT: v_add_u32_e32 v3, 0x80000000, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: ssubsat_i128_vs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s0
-; GFX10-NEXT: v_subrev_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: v_subrev_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX10-NEXT: v_subrev_co_ci_u32_e32 v4, vcc_lo, s3, v3, vcc_lo
-; GFX10-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX10-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
+; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, v0, s0
+; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT: v_subrev_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT: v_subrev_co_ci_u32_e32 v5, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[4:5]
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[4:5]
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v2
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v8, 0, vcc_lo
+; GFX10-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: ssubsat_i128_vs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s0
-; GFX11-NEXT: v_subrev_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX11-NEXT: v_subrev_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
-; GFX11-NEXT: v_subrev_co_ci_u32_e64 v4, null, s3, v3, vcc_lo
-; GFX11-NEXT: v_xor_b32_e32 v5, v3, v4
-; GFX11-NEXT: v_xor_b32_e32 v3, s3, v3
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v5
-; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, -v5, vcc_lo
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: ssubsat_i128_vs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_sub_co_u32 v6, vcc_lo, v0, s0
+; GFX11-TRUE16-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_subrev_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_subrev_co_ci_u32_e64 v5, null, s3, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e64 s0, s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s2, s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e64 s1, 0, v[4:5]
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[4:5]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x80000000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, 0, vcc_lo
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v6, v2 :: v_dual_cndmask_b32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: ssubsat_i128_vs:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_sub_co_u32 v6, vcc_lo, v0, s0
+; GFX11-FAKE16-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_subrev_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_subrev_co_ci_u32_e64 v5, null, s3, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[4:5]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v2, 31, v5
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v1, v0 :: v_dual_add_nc_u32 v3, 0x80000000, v2
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[4:5]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v8, 0, vcc_lo
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v6, v2 :: v_dual_cndmask_b32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i128 @llvm.ssub.sat.i128(i128 %lhs, i128 %rhs)
%cast = bitcast i128 %result to <4 x float>
ret <4 x float> %cast
@@ -5089,157 +5229,298 @@ define <2 x i128> @v_ssubsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
; GFX6-LABEL: v_ssubsat_v2i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
-; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v9, vcc
-; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v2, v10, vcc
-; GFX6-NEXT: v_subb_u32_e32 v8, vcc, v3, v11, vcc
-; GFX6-NEXT: v_xor_b32_e32 v9, v3, v8
-; GFX6-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX6-NEXT: v_and_b32_e32 v3, v3, v9
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v3, v8, -v9, vcc
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v12
-; GFX6-NEXT: v_subb_u32_e32 v5, vcc, v5, v13, vcc
-; GFX6-NEXT: v_subb_u32_e32 v6, vcc, v6, v14, vcc
-; GFX6-NEXT: v_subb_u32_e32 v8, vcc, v7, v15, vcc
-; GFX6-NEXT: v_xor_b32_e32 v9, v7, v8
-; GFX6-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX6-NEXT: v_and_b32_e32 v7, v7, v9
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 0, v7
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v7, v8, -v9, vcc
+; GFX6-NEXT: v_sub_i32_e32 v18, vcc, v0, v8
+; GFX6-NEXT: v_subb_u32_e32 v19, vcc, v1, v9, vcc
+; GFX6-NEXT: v_subb_u32_e32 v16, vcc, v2, v10, vcc
+; GFX6-NEXT: v_subb_u32_e32 v17, vcc, v3, v11, vcc
+; GFX6-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[10:11]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v17
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[16:17]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX6-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v2, v1
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v18, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v19, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, v4, v12
+; GFX6-NEXT: v_subb_u32_e32 v11, vcc, v5, v13, vcc
+; GFX6-NEXT: v_subb_u32_e32 v8, vcc, v6, v14, vcc
+; GFX6-NEXT: v_subb_u32_e32 v9, vcc, v7, v15, vcc
+; GFX6-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX6-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX6-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[14:15]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX6-NEXT: v_ashrrev_i32_e32 v6, 31, v9
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX6-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[8:9]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX6-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc
+; GFX6-NEXT: v_xor_b32_e32 v4, v4, v5
+; GFX6-NEXT: v_and_b32_e32 v4, 1, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0x80000000, v6
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v10, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, v11, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v8
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v1, v9, vcc
-; GFX8-NEXT: v_subb_u32_e32 v2, vcc, v2, v10, vcc
-; GFX8-NEXT: v_subb_u32_e32 v8, vcc, v3, v11, vcc
-; GFX8-NEXT: v_xor_b32_e32 v9, v3, v8
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v9
-; GFX8-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v8, -v9, vcc
-; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v4, v12
-; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v5, v13, vcc
-; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v6, v14, vcc
-; GFX8-NEXT: v_subb_u32_e32 v8, vcc, v7, v15, vcc
-; GFX8-NEXT: v_xor_b32_e32 v9, v7, v8
-; GFX8-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_and_b32_e32 v7, v7, v9
-; GFX8-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX8-NEXT: v_cmp_gt_i32_e32 vcc, 0, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v7, v8, -v9, vcc
+; GFX8-NEXT: v_sub_u32_e32 v18, vcc, v0, v8
+; GFX8-NEXT: v_subb_u32_e32 v19, vcc, v1, v9, vcc
+; GFX8-NEXT: v_subb_u32_e32 v16, vcc, v2, v10, vcc
+; GFX8-NEXT: v_subb_u32_e32 v17, vcc, v3, v11, vcc
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[10:11]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v17
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[16:17]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v2, v1
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v18, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v19, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
+; GFX8-NEXT: v_sub_u32_e32 v10, vcc, v4, v12
+; GFX8-NEXT: v_subb_u32_e32 v11, vcc, v5, v13, vcc
+; GFX8-NEXT: v_subb_u32_e32 v8, vcc, v6, v14, vcc
+; GFX8-NEXT: v_subb_u32_e32 v9, vcc, v7, v15, vcc
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[14:15]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX8-NEXT: v_ashrrev_i32_e32 v6, 31, v9
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc
+; GFX8-NEXT: v_xor_b32_e32 v4, v4, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x80000000, v6
+; GFX8-NEXT: v_cmp_ne_u16_e32 vcc, 0, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v10, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v11, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v8
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v9, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v10, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v3, v11, vcc
-; GFX9-NEXT: v_xor_b32_e32 v9, v3, v8
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v9
-; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, -v9, vcc
-; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v12
-; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v13, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v6, v14, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v7, v15, vcc
-; GFX9-NEXT: v_xor_b32_e32 v9, v7, v8
-; GFX9-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_and_b32_e32 v7, v7, v9
-; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v7
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v7, v8, -v9, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v18, vcc, v0, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v19, vcc, v1, v9, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v2, v10, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v3, v11, vcc
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[10:11]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[16:17]
+; GFX9-NEXT: v_add_u32_e32 v3, 0x80000000, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v19, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v10, vcc, v4, v12
+; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v5, v13, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v6, v14, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v7, v15, vcc
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[14:15]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX9-NEXT: v_ashrrev_i32_e32 v6, 31, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[8:9]
+; GFX9-NEXT: v_add_u32_e32 v7, 0x80000000, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v4, v4, v5
+; GFX9-NEXT: v_cmp_ne_u16_e32 vcc, 0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v10, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v11, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_ssubsat_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v8
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v9, vcc_lo
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v2, vcc_lo, v2, v10, vcc_lo
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v8, vcc_lo, v3, v11, vcc_lo
-; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v4, v12
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v5, vcc_lo, v5, v13, vcc_lo
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v6, vcc_lo, v6, v14, vcc_lo
+; GFX10-NEXT: v_sub_co_u32 v18, vcc_lo, v0, v8
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v19, vcc_lo, v1, v9, vcc_lo
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v16, vcc_lo, v2, v10, vcc_lo
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v17, vcc_lo, v3, v11, vcc_lo
+; GFX10-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[16:17]
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[16:17]
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v8, 0, vcc_lo
+; GFX10-NEXT: v_sub_co_u32 v10, vcc_lo, v4, v12
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v11, vcc_lo, v5, v13, vcc_lo
+; GFX10-NEXT: v_sub_co_ci_u32_e32 v8, vcc_lo, v6, v14, vcc_lo
; GFX10-NEXT: v_sub_co_ci_u32_e32 v9, vcc_lo, v7, v15, vcc_lo
-; GFX10-NEXT: v_xor_b32_e32 v10, v3, v8
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_xor_b32_e32 v11, v7, v9
-; GFX10-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v10
-; GFX10-NEXT: v_ashrrev_i32_e32 v10, 31, v8
-; GFX10-NEXT: v_and_b32_e32 v7, v7, v11
-; GFX10-NEXT: v_ashrrev_i32_e32 v11, 31, v9
-; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cmp_gt_i32_e64 s4, 0, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v8, -v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v11, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v11, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v11, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v9, -v11, s4
+; GFX10-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[12:13]
+; GFX10-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[6:7], v[14:15]
+; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[8:9]
+; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[14:15]
+; GFX10-NEXT: v_ashrrev_i32_e32 v6, 31, v9
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
+; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v17
+; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x80000000, v6
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc_lo
+; GFX10-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x80000000, v3
+; GFX10-NEXT: v_xor_b32_e32 v2, v1, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v18, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v19, v3, vcc_lo
+; GFX10-NEXT: v_cmp_ne_u16_e64 s4, 0, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v16, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v17, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v10, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v8, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v9, v7, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_ssubsat_v2i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v8
-; GFX11-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v9, vcc_lo
-; GFX11-NEXT: v_sub_co_ci_u32_e32 v2, vcc_lo, v2, v10, vcc_lo
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v8, null, v3, v11, vcc_lo
-; GFX11-NEXT: v_sub_co_u32 v4, vcc_lo, v4, v12
-; GFX11-NEXT: v_sub_co_ci_u32_e32 v5, vcc_lo, v5, v13, vcc_lo
-; GFX11-NEXT: v_sub_co_ci_u32_e32 v6, vcc_lo, v6, v14, vcc_lo
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v9, null, v7, v15, vcc_lo
-; GFX11-NEXT: v_xor_b32_e32 v10, v3, v8
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX11-NEXT: v_xor_b32_e32 v11, v7, v9
-; GFX11-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v10
-; GFX11-NEXT: v_ashrrev_i32_e32 v10, 31, v8
-; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX11-NEXT: v_and_b32_e32 v7, v7, v11
-; GFX11-NEXT: v_ashrrev_i32_e32 v11, 31, v9
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
-; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v7
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v8, -v10, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v11, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v11, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v11, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v9, -v11, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_ssubsat_v2i128:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_sub_co_u32 v16, vcc_lo, v0, v8
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e32 v17, vcc_lo, v1, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_u64_e64 s0, v[0:1], v[8:9]
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e32 v8, vcc_lo, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e64 v9, null, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_i64_e64 s1, v[2:3], v[10:11]
+; GFX11-TRUE16-NEXT: v_sub_co_u32 v18, s2, v4, v12
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e64 v19, s2, v5, v13, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[10:11]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e64 s0, 0, v[8:9]
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e64 v10, s2, v6, v14, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s1
+; GFX11-TRUE16-NEXT: v_sub_co_ci_u32_e64 v11, null, v7, v15, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s1, 0, v[8:9]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_u64_e64 s0, v[4:5], v[12:13]
+; GFX11-TRUE16-NEXT: v_cmp_lt_i64_e64 s2, v[6:7], v[14:15]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[10:11]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, 0, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s1, v[6:7], v[14:15]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, 0, v[10:11]
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, 0, 1, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v9
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v6, 31, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x80000000, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x80000000, v6
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v16, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v17, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v2.l
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v3 :: v_dual_cndmask_b32 v3, v9, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, v18, v6, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v5, v19, v6, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v6, v10, v6, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v7, v11, v7, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_ssubsat_v2i128:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_sub_co_u32 v18, vcc_lo, v0, v8
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e32 v19, vcc_lo, v1, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e32 v16, vcc_lo, v2, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e64 v17, null, v3, v11, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[10:11]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[16:17]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[10:11]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[16:17]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v8, 0, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_co_u32 v10, vcc_lo, v4, v12
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e32 v11, vcc_lo, v5, v13, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e32 v8, vcc_lo, v6, v14, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_co_ci_u32_e64 v9, null, v7, v15, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[12:13]
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[6:7], v[14:15]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[8:9]
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[14:15]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v6, 31, v9
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v17
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x80000000, v6
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x80000000, v3
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, v1, v2
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v18, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v19, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v2
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v2, v16, v3 :: v_dual_cndmask_b32 v3, v17, v4
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v10, v6, s0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v5, v11, v6, s0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, v8, v6, s0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v7, v9, v7, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
ret <2 x i128> %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index 2ef329be39c76..f13c387042b08 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck --check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX11 %s
; FIXME:
; XUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck --check-prefix=GFX6 %s
@@ -70,7 +70,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x100000
+; GFX9-NEXT: s_and_b32 s5, 0xffff, s0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: s_lshr_b32 s5, s5, 8
@@ -82,7 +82,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: ds_write_b8 v1, v0 offset:2
; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: s_bfe_u32 s4, s1, 0x100000
+; GFX9-NEXT: s_and_b32 s4, 0xffff, s1
; GFX9-NEXT: ds_write_b8 v1, v0 offset:3
; GFX9-NEXT: s_lshr_b32 s4, s4, 8
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -94,7 +94,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: ds_write_b8 v1, v0 offset:6
; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: s_bfe_u32 s1, s2, 0x100000
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s2
; GFX9-NEXT: ds_write_b8 v1, v0 offset:7
; GFX9-NEXT: s_lshr_b32 s1, s1, 8
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -106,7 +106,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: ds_write_b8 v1, v0 offset:10
; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: s_bfe_u32 s1, s3, 0x100000
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s3
; GFX9-NEXT: ds_write_b8 v1, v0 offset:11
; GFX9-NEXT: s_lshr_b32 s1, s1, 8
; GFX9-NEXT: v_mov_b32_e32 v0, s3
@@ -183,9 +183,9 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX10-NEXT: s_lshr_b32 s4, s0, 16
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s6
-; GFX10-NEXT: s_bfe_u32 s5, s0, 0x100000
+; GFX10-NEXT: s_and_b32 s5, 0xffff, s0
; GFX10-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-NEXT: s_bfe_u32 s7, s1, 0x100000
+; GFX10-NEXT: s_and_b32 s7, 0xffff, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: s_lshr_b32 s6, s1, 16
; GFX10-NEXT: v_mov_b32_e32 v4, s4
@@ -205,14 +205,14 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX10-NEXT: ds_write_b8 v1, v9 offset:5
; GFX10-NEXT: v_mov_b32_e32 v0, s8
; GFX10-NEXT: s_lshr_b32 s1, s1, 24
-; GFX10-NEXT: s_bfe_u32 s9, s2, 0x100000
+; GFX10-NEXT: s_and_b32 s9, 0xffff, s2
; GFX10-NEXT: s_lshr_b32 s0, s2, 24
; GFX10-NEXT: v_mov_b32_e32 v7, s1
; GFX10-NEXT: s_lshr_b32 s1, s9, 8
; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: ds_write_b8 v1, v0 offset:10
; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_bfe_u32 s0, s3, 0x100000
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s3
; GFX10-NEXT: v_mov_b32_e32 v10, s1
; GFX10-NEXT: s_lshr_b32 s0, s0, 8
; GFX10-NEXT: s_lshr_b32 s1, s3, 16
@@ -234,41 +234,58 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-LABEL: store_lds_v4i32_align1:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x0
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
+; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s3
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: s_lshr_b32 s4, s3, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s4
-; GFX11-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-NEXT: s_lshr_b32 s2, s2, 24
-; GFX11-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-NEXT: s_lshr_b32 s3, s3, 24
-; GFX11-NEXT: s_lshr_b32 s1, s1, 24
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-NEXT: s_and_b32 s6, 0xffff, s0
+; GFX11-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s6
-; GFX11-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s5
-; GFX11-NEXT: v_mov_b32_e32 v10, s1
-; GFX11-NEXT: ds_store_b8 v0, v2 offset:8
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v2 offset:10
-; GFX11-NEXT: ds_store_b8 v0, v1 offset:12
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v1 offset:14
-; GFX11-NEXT: ds_store_b8 v0, v5 offset:13
-; GFX11-NEXT: ds_store_b8 v0, v6 offset:15
-; GFX11-NEXT: ds_store_b8 v0, v7 offset:9
-; GFX11-NEXT: ds_store_b8 v0, v8 offset:11
-; GFX11-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: ds_store_b8 v0, v4
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v4 offset:2
-; GFX11-NEXT: ds_store_b8 v0, v3 offset:4
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v3 offset:6
-; GFX11-NEXT: ds_store_b8 v0, v9 offset:5
-; GFX11-NEXT: ds_store_b8 v0, v10 offset:7
-; GFX11-NEXT: ds_store_b8 v0, v1 offset:1
-; GFX11-NEXT: ds_store_b8 v0, v2 offset:3
+; GFX11-NEXT: s_and_b32 s7, 0xffff, s1
+; GFX11-NEXT: s_lshr_b32 s6, s6, 8
+; GFX11-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-NEXT: s_lshr_b32 s0, s7, 8
+; GFX11-NEXT: v_mov_b16_e32 v3.l, s6
+; GFX11-NEXT: v_mov_b16_e32 v1.h, s5
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 24
+; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
+; GFX11-NEXT: v_mov_b16_e32 v4.l, s0
+; GFX11-NEXT: s_lshr_b32 s0, s2, 24
+; GFX11-NEXT: v_mov_b16_e32 v2.h, s4
+; GFX11-NEXT: v_mov_b16_e32 v3.h, s1
+; GFX11-NEXT: s_lshr_b32 s1, s9, 8
+; GFX11-NEXT: ds_store_b8 v5, v0
+; GFX11-NEXT: ds_store_b8 v5, v3 offset:1
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v1 offset:2
+; GFX11-NEXT: ds_store_b8 v5, v2 offset:3
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v0 offset:4
+; GFX11-NEXT: ds_store_b8 v5, v4 offset:5
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v2 offset:6
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:7
+; GFX11-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s3
+; GFX11-NEXT: s_lshr_b32 s8, s2, 16
+; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX11-NEXT: v_mov_b16_e32 v0.l, s1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-NEXT: v_mov_b16_e32 v0.h, s8
+; GFX11-NEXT: s_lshr_b32 s1, s3, 16
+; GFX11-NEXT: v_mov_b16_e32 v2.h, s0
+; GFX11-NEXT: s_lshr_b32 s0, s3, 24
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s3
+; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX11-NEXT: v_mov_b16_e32 v3.h, s0
+; GFX11-NEXT: ds_store_b8 v5, v1 offset:8
+; GFX11-NEXT: ds_store_b8 v5, v0 offset:9
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v0 offset:10
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v1 offset:11
+; GFX11-NEXT: ds_store_b8 v5, v2 offset:12
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v2 offset:13
+; GFX11-NEXT: ds_store_b8 v5, v3 offset:14
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:15
; GFX11-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 1
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index ed849842cdfcf..8cc2985728255 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck --check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX11 %s
; FIXME:
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck --check-prefix=GFX6 %s
@@ -88,7 +88,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX9-NEXT: ; kill: killed $sgpr4_sgpr5
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x100000
+; GFX9-NEXT: s_and_b32 s5, 0xffff, s0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s3
@@ -101,7 +101,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: ds_write_b8 v1, v0 offset:2
; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: s_bfe_u32 s3, s1, 0x100000
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s1
; GFX9-NEXT: ds_write_b8 v1, v0 offset:3
; GFX9-NEXT: s_lshr_b32 s3, s3, 8
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -113,7 +113,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: ds_write_b8 v1, v0 offset:6
; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: s_bfe_u32 s1, s2, 0x100000
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s2
; GFX9-NEXT: ds_write_b8 v1, v0 offset:7
; GFX9-NEXT: s_lshr_b32 s1, s1, 8
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -178,16 +178,16 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX10-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX10-NEXT: ; kill: killed $sgpr4_sgpr5
; GFX10-NEXT: s_lshr_b32 s4, s0, 16
-; GFX10-NEXT: s_bfe_u32 s5, s0, 0x100000
+; GFX10-NEXT: s_and_b32 s5, 0xffff, s0
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_lshr_b32 s0, s0, 24
; GFX10-NEXT: s_lshr_b32 s3, s1, 16
-; GFX10-NEXT: s_bfe_u32 s6, s1, 0x100000
+; GFX10-NEXT: s_and_b32 s6, 0xffff, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: s_lshr_b32 s1, s1, 24
-; GFX10-NEXT: s_bfe_u32 s8, s2, 0x100000
+; GFX10-NEXT: s_and_b32 s8, 0xffff, s2
; GFX10-NEXT: v_mov_b32_e32 v4, s4
; GFX10-NEXT: v_mov_b32_e32 v5, s0
; GFX10-NEXT: s_lshr_b32 s0, s6, 8
@@ -219,34 +219,47 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
;
; GFX11-LABEL: store_lds_v3i32_align1:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x0
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: s_lshr_b32 s3, s2, 8
-; GFX11-NEXT: s_lshr_b32 s2, s2, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s2
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-NEXT: s_lshr_b32 s4, s1, 8
-; GFX11-NEXT: s_lshr_b32 s1, s1, 24
-; GFX11-NEXT: s_lshr_b32 s5, s0, 8
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX11-NEXT: s_and_b32 s5, 0xffff, s0
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s5, 8
+; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, s3
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
-; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s1
-; GFX11-NEXT: v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v9, s0
-; GFX11-NEXT: ds_store_b8 v0, v1 offset:8
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v1 offset:10
-; GFX11-NEXT: ds_store_b8 v0, v4 offset:9
-; GFX11-NEXT: ds_store_b8 v0, v5 offset:11
-; GFX11-NEXT: ds_store_b8 v0, v3
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v3 offset:2
-; GFX11-NEXT: ds_store_b8 v0, v2 offset:4
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v2 offset:6
-; GFX11-NEXT: ds_store_b8 v0, v6 offset:5
-; GFX11-NEXT: ds_store_b8 v0, v7 offset:7
-; GFX11-NEXT: ds_store_b8 v0, v8 offset:1
-; GFX11-NEXT: ds_store_b8 v0, v9 offset:3
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 24
+; GFX11-NEXT: s_and_b32 s8, 0xffff, s2
+; GFX11-NEXT: v_mov_b16_e32 v4.l, s5
+; GFX11-NEXT: v_mov_b16_e32 v1.h, s4
+; GFX11-NEXT: s_lshr_b32 s7, s2, 16
+; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX11-NEXT: s_lshr_b32 s2, s2, 24
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-NEXT: s_lshr_b32 s0, s6, 8
+; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX11-NEXT: s_lshr_b32 s1, s8, 8
+; GFX11-NEXT: v_mov_b16_e32 v2.h, s3
+; GFX11-NEXT: v_mov_b16_e32 v3.h, s7
+; GFX11-NEXT: v_mov_b16_e32 v4.h, s2
+; GFX11-NEXT: v_mov_b16_e32 v5.l, s0
+; GFX11-NEXT: v_mov_b16_e32 v5.h, s1
+; GFX11-NEXT: ds_store_b8 v6, v0
+; GFX11-NEXT: ds_store_b8 v6, v4 offset:1
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v1 offset:2
+; GFX11-NEXT: ds_store_b8 v6, v2 offset:3
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v0 offset:4
+; GFX11-NEXT: ds_store_b8 v6, v5 offset:5
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v2 offset:6
+; GFX11-NEXT: ds_store_b8 v6, v3 offset:7
+; GFX11-NEXT: ds_store_b8 v6, v1 offset:8
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v5 offset:9
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v3 offset:10
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v4 offset:11
; GFX11-NEXT: s_endpgm
;
; GFX6-LABEL: store_lds_v3i32_align1:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
index ea71106c677d9..768a8976ced54 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
@@ -167,9 +167,17 @@ define void @v_constained_fma_v2f16_fpexcept_strict_uni(<2 x half> inreg %x, <2
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_fma_f16 v2, s0, s1, v2
+; GFX12-NEXT: s_lshr_b32 s3, s0, 16
+; GFX12-NEXT: s_lshr_b32 s4, s1, 16
+; GFX12-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-NEXT: s_fmac_f16 s2, s0, s1
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_fmac_f16 s5, s3, s4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %x, <2 x half> %y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -371,20 +379,21 @@ define void @v_constained_fma_v4f16_fpexcept_strict_uni(<4 x half> inreg %x, <4
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s4, s17, 16
-; GFX12-NEXT: s_lshr_b32 s5, s3, 16
-; GFX12-NEXT: s_lshr_b32 s6, s1, 16
-; GFX12-NEXT: s_lshr_b32 s7, s16, 16
-; GFX12-NEXT: s_lshr_b32 s8, s2, 16
-; GFX12-NEXT: s_lshr_b32 s9, s0, 16
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_fmac_f16 s4, s6, s5
-; GFX12-NEXT: s_fmac_f16 s7, s9, s8
+; GFX12-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-NEXT: s_lshr_b32 s6, s16, 16
; GFX12-NEXT: s_fmac_f16 s16, s0, s2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_fmac_f16 s6, s4, s5
+; GFX12-NEXT: s_lshr_b32 s0, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s3, 16
+; GFX12-NEXT: s_lshr_b32 s4, s17, 16
; GFX12-NEXT: s_fmac_f16 s17, s1, s3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_pack_ll_b32_b16 s0, s16, s7
+; GFX12-NEXT: s_fmac_f16 s4, s0, s2
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s16, s6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_pack_ll_b32_b16 s1, s17, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
@@ -844,9 +853,22 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni(<2 x half> inr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_fma_f16 v2, s0, s1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX12-NEXT: s_lshr_b32 s3, s0, 16
+; GFX12-NEXT: s_lshr_b32 s4, s1, 16
+; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_xor_b32 s3, s3, 0x8000
+; GFX12-NEXT: s_xor_b32 s1, s1, 0x8000
+; GFX12-NEXT: s_xor_b32 s4, s4, 0x8000
+; GFX12-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_fmac_f16 s2, s0, s1
+; GFX12-NEXT: s_fmac_f16 s5, s3, s4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg <2 x half> %x
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
index 39540289eb138..8854302456a20 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
define <2 x i16> @v_sub_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX9-LABEL: v_sub_v2i16:
@@ -235,9 +235,10 @@ define <2 x i16> @v_sub_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
; GFX9-LABEL: s_sub_v2i16_neg_inline_imm_splat:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_not_b32_e32 v0, 63
-; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_sub_i32 s0, s0, 0xffc0ffc0
+; GFX9-NEXT: s_sub_i32 s1, s1, 0xffc0
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sub_v2i16_neg_inline_imm_splat:
@@ -251,14 +252,18 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
;
; GFX10-LABEL: s_sub_v2i16_neg_inline_imm_splat:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_sub_i16 v0, s0, 0xffffffc0 op_sel_hi:[1,0]
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-NEXT: s_sub_i32 s0, s0, 0xffc0ffc0
+; GFX10-NEXT: s_sub_i32 s1, s1, 0xffc0
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_sub_v2i16_neg_inline_imm_splat:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_sub_i16 v0, s0, 0xffffffc0 op_sel_hi:[1,0]
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-NEXT: s_sub_i32 s0, s0, 0xffc0ffc0
+; GFX11-NEXT: s_sub_i32 s1, s1, 0xffc0
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%sub = sub <2 x i16> %a, <i16 -64, i16 -64>
%cast = bitcast <2 x i16> %sub to i32
@@ -268,9 +273,10 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
; GFX9-LABEL: s_sub_v2i16_neg_inline_imm_lo:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x4ffc0
-; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_sub_i32 s0, s0, 0x4ffc0
+; GFX9-NEXT: s_sub_i32 s1, s1, 4
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sub_v2i16_neg_inline_imm_lo:
@@ -284,14 +290,18 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
;
; GFX10-LABEL: s_sub_v2i16_neg_inline_imm_lo:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_sub_i16 v0, s0, 0x4ffc0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-NEXT: s_sub_i32 s0, s0, 0x4ffc0
+; GFX10-NEXT: s_sub_i32 s1, s1, 4
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_sub_v2i16_neg_inline_imm_lo:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_sub_i16 v0, s0, 0x4ffc0
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-NEXT: s_sub_i32 s0, s0, 0x4ffc0
+; GFX11-NEXT: s_sub_i32 s1, s1, 4
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%sub = sub <2 x i16> %a, <i16 -64, i16 4>
%cast = bitcast <2 x i16> %sub to i32
@@ -301,9 +311,10 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
; GFX9-LABEL: s_sub_v2i16_neg_inline_imm_hi:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, 0xffc00004
-; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_sub_i32 s0, s0, 0xffc00004
+; GFX9-NEXT: s_sub_i32 s1, s1, 0xffc0
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sub_v2i16_neg_inline_imm_hi:
@@ -317,14 +328,18 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
;
; GFX10-LABEL: s_sub_v2i16_neg_inline_imm_hi:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_sub_i16 v0, s0, 0xffc00004
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-NEXT: s_sub_i32 s0, s0, 0xffc00004
+; GFX10-NEXT: s_sub_i32 s1, s1, 0xffc0
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_sub_v2i16_neg_inline_imm_hi:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_sub_i16 v0, s0, 0xffc00004
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-NEXT: s_sub_i32 s0, s0, 0xffc00004
+; GFX11-NEXT: s_sub_i32 s1, s1, 0xffc0
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%sub = sub <2 x i16> %a, <i16 4, i16 -64>
%cast = bitcast <2 x i16> %sub to i32
@@ -334,9 +349,11 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
define amdgpu_ps i32 @s_sub_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
; GFX9-LABEL: s_sub_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s1, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sub_v2i16:
@@ -352,14 +369,20 @@ define amdgpu_ps i32 @s_sub_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
;
; GFX10-LABEL: s_sub_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_pk_sub_i16 v0, s0, s1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_sub_i32 s0, s0, s1
+; GFX10-NEXT: s_sub_i32 s1, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_sub_v2i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_sub_i16 v0, s0, s1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-NEXT: s_sub_i32 s1, s2, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%sub = sub <2 x i16> %a, %b
%cast = bitcast <2 x i16> %sub to i32
@@ -369,10 +392,14 @@ define amdgpu_ps i32 @s_sub_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
define amdgpu_ps i32 @s_sub_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg %b) {
; GFX9-LABEL: s_sub_v2i16_fneg_lhs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s1, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sub_v2i16_fneg_lhs:
@@ -389,16 +416,24 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg
;
; GFX10-LABEL: s_sub_v2i16_fneg_lhs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX10-NEXT: v_pk_sub_i16 v0, s0, s1
+; GFX10-NEXT: v_xor_b32_e64 v0, 0x80008000, s0
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_sub_i32 s0, s0, s1
+; GFX10-NEXT: s_sub_i32 s1, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_sub_v2i16_fneg_lhs:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX11-NEXT: v_pk_sub_i16 v0, s0, s1
+; GFX11-NEXT: v_xor_b32_e64 v0, 0x80008000, s0
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-NEXT: s_sub_i32 s1, s2, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%neg.a = fneg <2 x half> %a
%cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -410,10 +445,14 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg
define amdgpu_ps i32 @s_sub_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg %b) {
; GFX9-LABEL: s_sub_v2i16_fneg_rhs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s1, 0x80008000
; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s1, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sub_v2i16_fneg_rhs:
@@ -430,16 +469,24 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg
;
; GFX10-LABEL: s_sub_v2i16_fneg_rhs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX10-NEXT: v_pk_sub_i16 v0, s0, s1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_xor_b32_e64 v0, 0x80008000, s1
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: v_readfirstlane_b32 s1, v0
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_sub_i32 s0, s0, s1
+; GFX10-NEXT: s_sub_i32 s1, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_sub_v2i16_fneg_rhs:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX11-NEXT: v_pk_sub_i16 v0, s0, s1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_xor_b32_e64 v0, 0x80008000, s1
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: v_readfirstlane_b32 s1, v0
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-NEXT: s_sub_i32 s1, s2, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%neg.b = fneg <2 x half> %b
%cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
@@ -451,11 +498,17 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg
define amdgpu_ps i32 @s_sub_v2i16_fneg_lhs_fneg_rhs(<2 x half> inreg %a, <2 x half> inreg %b) {
; GFX9-LABEL: s_sub_v2i16_fneg_lhs_fneg_rhs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX9-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s1, s2, s3
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sub_v2i16_fneg_lhs_fneg_rhs:
@@ -473,18 +526,28 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_lhs_fneg_rhs(<2 x half> inreg %a, <2 x ha
;
; GFX10-LABEL: s_sub_v2i16_fneg_lhs_fneg_rhs:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX10-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX10-NEXT: v_pk_sub_i16 v0, s0, s1
+; GFX10-NEXT: v_xor_b32_e64 v0, 0x80008000, s0
+; GFX10-NEXT: v_xor_b32_e64 v1, 0x80008000, s1
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_sub_i32 s0, s0, s1
+; GFX10-NEXT: s_sub_i32 s1, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_sub_v2i16_fneg_lhs_fneg_rhs:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX11-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX11-NEXT: v_pk_sub_i16 v0, s0, s1
+; GFX11-NEXT: v_xor_b32_e64 v0, 0x80008000, s0
+; GFX11-NEXT: v_xor_b32_e64 v1, 0x80008000, s1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-NEXT: s_sub_i32 s1, s2, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%neg.a = fneg <2 x half> %a
%neg.b = fneg <2 x half> %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/true16-merge-values-s16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/true16-merge-values-s16.ll
index 19378f2807f1d..e03c164803b51 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/true16-merge-values-s16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/true16-merge-values-s16.ll
@@ -1,30 +1,28 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=+real-true16 -o - %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -mattr=+real-true16 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=+real-true16 -o - %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -mattr=+real-true16 -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps i48 @test_merge_values_sgpr(<3 x i16> inreg %src0, <3 x i16> inreg %src1) {
; GFX11-LABEL: test_merge_values_sgpr:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 s0, s1, s0
-; GFX11-NEXT: s_and_not1_b32 s1, s2, s4
-; GFX11-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_and_b32 s1, s3, s2
+; GFX11-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: test_merge_values_sgpr:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_lshr_b32 s0, s4, 16
-; GFX12-NEXT: s_lshr_b32 s1, s2, 16
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 s0, s1, s0
-; GFX12-NEXT: s_and_not1_b32 s1, s2, s4
-; GFX12-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX12-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX12-NEXT: s_and_b32 s1, s3, s2
+; GFX12-NEXT: s_mov_b64 s[0:1], -1
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX12-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s2
; GFX12-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%and = and <3 x i16> %src0, %not.src1
@@ -35,31 +33,25 @@ define amdgpu_ps i48 @test_merge_values_sgpr(<3 x i16> inreg %src0, <3 x i16> in
define amdgpu_ps i48 @test_merge_values_vgpr(<3 x i16> %src0, <3 x i16> %src1) {
; GFX11-LABEL: test_merge_values_vgpr:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_xor_b16 v1.h, v2.h, -1
-; GFX11-NEXT: v_xor_b16 v2.l, v2.l, -1
-; GFX11-NEXT: v_xor_b16 v2.h, v3.l, -1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, v1.h
-; GFX11-NEXT: v_and_b16 v0.l, v0.l, v2.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b16 v1.l, v1.l, v2.h
+; GFX11-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX11-NEXT: v_xor_b32_e32 v3, -1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: test_merge_values_vgpr:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_xor_b16 v1.h, v2.h, -1
-; GFX12-NEXT: v_xor_b16 v2.l, v2.l, -1
-; GFX12-NEXT: v_xor_b16 v2.h, v3.l, -1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_and_b16 v0.h, v0.h, v1.h
-; GFX12-NEXT: v_and_b16 v0.l, v0.l, v2.l
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b16 v1.l, v1.l, v2.h
+; GFX12-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX12-NEXT: v_xor_b32_e32 v3, -1, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index c649208502090..62653a20cea78 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -79,30 +79,55 @@ define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
;
; GFX8-LABEL: s_uaddsat_i7:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX8-NEXT: s_and_b32 s0, s0, 0x7f
-; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_min_u32 s0, s0, 0x7f
+; GFX8-NEXT: s_lshl_b32 s1, s1, 9
+; GFX8-NEXT: s_lshl_b32 s0, s0, 9
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 9
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_i7:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX9-NEXT: s_and_b32 s0, s0, 0x7f
-; GFX9-NEXT: s_add_i32 s0, s0, s1
-; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_min_u32 s0, s0, 0x7f
+; GFX9-NEXT: s_lshl_b32 s1, s1, 9
+; GFX9-NEXT: s_lshl_b32 s0, s0, 9
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 9
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_uaddsat_i7:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0x7f
-; GFX10PLUS-NEXT: s_add_i32 s0, s0, s1
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_min_u32 s0, s0, 0x7f
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_uaddsat_i7:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b32 s0, s0, 9
+; GFX10-NEXT: s_lshl_b32 s1, s1, 9
+; GFX10-NEXT: v_add_nc_u16 v0, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_lshr_b32 s0, s0, 9
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_uaddsat_i7:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 9
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 9
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_uaddsat_i7:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 9
+; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 9
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i7 @llvm.uadd.sat.i7(i7 %lhs, i7 %rhs)
ret i7 %result
}
@@ -180,30 +205,55 @@ define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
;
; GFX8-LABEL: s_uaddsat_i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s1, 0xff
-; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX8-NEXT: s_min_u32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_add_i32 s0, s0, s1
-; GFX9-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX9-NEXT: s_min_u32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_uaddsat_i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_add_i32 s0, s0, s1
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX10PLUS-NEXT: s_min_u32 s0, s0, 0xff
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_uaddsat_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b32 s0, s0, 8
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10-NEXT: v_add_nc_u16 v0, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_uaddsat_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_uaddsat_i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i8 @llvm.uadd.sat.i8(i8 %lhs, i8 %rhs)
ret i8 %result
}
@@ -328,18 +378,22 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX8-LABEL: s_uaddsat_v2i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX8-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 8
-; GFX8-NEXT: s_lshr_b32 s3, s3, 8
-; GFX8-NEXT: s_and_b32 s1, s1, 0xff
-; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_add_i32 s3, s3, s2
-; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s1, s3, 0x1ff
-; GFX8-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX8-NEXT: s_min_u32 s1, s1, 0xff
-; GFX8-NEXT: s_min_u32 s0, s0, 0xff
+; GFX8-NEXT: s_lshr_b32 s3, s1, 8
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_add_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s1, s1, 8
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
; GFX8-NEXT: s_lshl_b32 s1, s1, 8
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
@@ -362,23 +416,67 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_uaddsat_v2i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX10PLUS-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX10PLUS-NEXT: s_lshr_b32 s2, s2, 8
-; GFX10PLUS-NEXT: s_lshr_b32 s3, s3, 8
-; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_add_i32 s3, s3, s2
-; GFX10PLUS-NEXT: s_add_i32 s0, s0, s1
-; GFX10PLUS-NEXT: s_and_b32 s1, s3, 0x1ff
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX10PLUS-NEXT: s_min_u32 s1, s1, 0xff
-; GFX10PLUS-NEXT: s_min_u32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 8
-; GFX10PLUS-NEXT: s_or_b32 s0, s0, s1
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_uaddsat_v2i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s2, 0xffff, s1
+; GFX10-NEXT: s_and_b32 s3, 0xffff, s0
+; GFX10-NEXT: s_lshr_b32 s2, s2, 8
+; GFX10-NEXT: s_lshr_b32 s3, s3, 8
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_add_i32 s3, s3, s2
+; GFX10-NEXT: s_add_i32 s0, s0, s1
+; GFX10-NEXT: s_and_b32 s1, s3, 0x1ff
+; GFX10-NEXT: s_and_b32 s0, s0, 0x1ff
+; GFX10-NEXT: s_min_u32 s1, s1, 0xff
+; GFX10-NEXT: s_min_u32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_uaddsat_v2i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_uaddsat_v2i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s2, 0xffff, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s3, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_add_i32 s3, s3, s2
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s3, 0x1ff
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x1ff
+; GFX11-FAKE16-NEXT: s_min_u32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_min_u32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%result = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ubfx.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ubfx.ll
index 3a1faa5c813d5..394f9fad6fb28 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ubfx.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ubfx.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - < %s | FileCheck --check-prefixes=GCN %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - < %s | FileCheck --check-prefixes=GCN %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - < %s | FileCheck --check-prefixes=GCN %s
; Test vector bitfield extract.
define i32 @v_srl_mask_i32(i32 %value) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
index 7aafc424f25ae..029bcc06e6f85 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,GISEL %s
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,CGP %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,CGP %s
; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
@@ -8,147 +8,173 @@ define i64 @v_udiv_i64(i64 %num, i64 %den) {
; CHECK-LABEL: v_udiv_i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_or_b32_e32 v4, v1, v3
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: v_mov_b32_e32 v5, v1
+; CHECK-NEXT: v_cvt_f32_u32_e32 v6, v2
+; CHECK-NEXT: v_mov_b32_e32 v4, v0
+; CHECK-NEXT: v_or_b32_e32 v1, v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB0_2
-; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v4, v2
-; CHECK-NEXT: v_cvt_f32_u32_e32 v5, v3
+; CHECK-NEXT: s_cbranch_execnz .LBB0_3
+; CHECK-NEXT: ; %bb.1: ; %Flow
+; CHECK-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_4
+; CHECK-NEXT: .LBB0_2: ; %.split
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; CHECK-NEXT: .LBB0_3:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, v3
+; CHECK-NEXT: v_mac_f32_e32 v6, 0x4f800000, v0
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v6
; CHECK-NEXT: v_sub_i32_e32 v6, vcc, 0, v2
; CHECK-NEXT: v_subb_u32_e32 v7, vcc, 0, v3, vcc
-; CHECK-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CHECK-NEXT: v_rcp_f32_e32 v4, v4
-; CHECK-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CHECK-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CHECK-NEXT: v_trunc_f32_e32 v5, v5
-; CHECK-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CHECK-NEXT: v_cvt_u32_f32_e32 v5, v5
-; CHECK-NEXT: v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT: v_mul_lo_u32 v8, v6, v5
-; CHECK-NEXT: v_mul_hi_u32 v9, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v11, v7, v4
-; CHECK-NEXT: v_mul_lo_u32 v10, v6, v4
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; CHECK-NEXT: v_trunc_f32_e32 v1, v1
+; CHECK-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v8, v6, v1
+; CHECK-NEXT: v_mul_hi_u32 v9, v6, v0
+; CHECK-NEXT: v_mul_lo_u32 v11, v7, v0
+; CHECK-NEXT: v_mul_lo_u32 v10, v6, v0
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; CHECK-NEXT: v_mul_hi_u32 v9, v4, v10
-; CHECK-NEXT: v_mul_lo_u32 v11, v4, v8
-; CHECK-NEXT: v_mul_hi_u32 v12, v4, v8
-; CHECK-NEXT: v_mul_hi_u32 v13, v5, v8
-; CHECK-NEXT: v_mul_lo_u32 v8, v5, v8
+; CHECK-NEXT: v_mul_hi_u32 v9, v0, v10
+; CHECK-NEXT: v_mul_lo_u32 v11, v0, v8
+; CHECK-NEXT: v_mul_lo_u32 v12, v1, v10
+; CHECK-NEXT: v_mul_hi_u32 v10, v1, v10
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v11
-; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
-; CHECK-NEXT: v_mul_lo_u32 v12, v5, v10
-; CHECK-NEXT: v_mul_hi_u32 v10, v5, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, v11, v10, vcc
-; CHECK-NEXT: v_addc_u32_e32 v10, vcc, 0, v13, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v12, v0, v8
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CHECK-NEXT: v_mul_lo_u32 v11, v1, v8
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CHECK-NEXT: v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc
+; CHECK-NEXT: v_mul_hi_u32 v8, v6, v0
+; CHECK-NEXT: v_mul_lo_u32 v9, v6, v1
+; CHECK-NEXT: v_mul_lo_u32 v7, v7, v0
+; CHECK-NEXT: v_mul_lo_u32 v6, v6, v0
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT: v_mul_hi_u32 v8, v0, v6
+; CHECK-NEXT: v_mul_lo_u32 v9, v0, v7
+; CHECK-NEXT: v_mul_lo_u32 v10, v1, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v1, v6
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v10, v0, v7
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc
-; CHECK-NEXT: v_mul_lo_u32 v8, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v9, v6, v5
-; CHECK-NEXT: v_mul_hi_u32 v6, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v7, v7, v4
-; CHECK-NEXT: v_mul_hi_u32 v10, v5, v8
+; CHECK-NEXT: v_mul_lo_u32 v9, v1, v7
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT: v_mul_lo_u32 v7, v4, v6
-; CHECK-NEXT: v_mul_hi_u32 v9, v4, v8
-; CHECK-NEXT: v_mul_hi_u32 v11, v4, v6
-; CHECK-NEXT: v_mul_lo_u32 v8, v5, v8
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v9, v7
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
-; CHECK-NEXT: v_mul_hi_u32 v11, v5, v6
-; CHECK-NEXT: v_mul_lo_u32 v6, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT: v_mul_hi_u32 v7, v1, v7
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; CHECK-NEXT: v_addc_u32_e32 v7, vcc, v9, v10, vcc
-; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v6
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc
-; CHECK-NEXT: v_mul_lo_u32 v6, v0, v5
-; CHECK-NEXT: v_mul_hi_u32 v7, v0, v4
-; CHECK-NEXT: v_mul_hi_u32 v8, v0, v5
-; CHECK-NEXT: v_mul_hi_u32 v9, v1, v5
-; CHECK-NEXT: v_mul_lo_u32 v5, v1, v5
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; CHECK-NEXT: v_mul_lo_u32 v8, v1, v4
-; CHECK-NEXT: v_mul_hi_u32 v4, v1, v4
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
+; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, v0
+; CHECK-NEXT: v_mul_lo_u32 v7, v4, v1
+; CHECK-NEXT: v_mul_lo_u32 v8, v5, v0
+; CHECK-NEXT: v_mul_hi_u32 v0, v5, v0
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v7, v4, vcc
-; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v9, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; CHECK-NEXT: v_mul_lo_u32 v6, v2, v5
-; CHECK-NEXT: v_mul_hi_u32 v7, v2, v4
-; CHECK-NEXT: v_mul_lo_u32 v8, v3, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v8, v4, v1
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CHECK-NEXT: v_mul_lo_u32 v7, v2, v4
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT: v_sub_i32_e32 v8, vcc, v1, v6
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
-; CHECK-NEXT: v_subb_u32_e64 v7, s[4:5], v8, v3, vcc
-; CHECK-NEXT: v_sub_i32_e64 v8, s[4:5], v0, v2
-; CHECK-NEXT: v_subbrev_u32_e64 v7, s[4:5], 0, v7, s[4:5]
+; CHECK-NEXT: v_mul_lo_u32 v7, v5, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT: v_mul_hi_u32 v1, v5, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v2, v0
+; CHECK-NEXT: v_mul_lo_u32 v7, v2, v1
+; CHECK-NEXT: v_mul_lo_u32 v9, v3, v0
+; CHECK-NEXT: v_mul_lo_u32 v8, v2, v0
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v9
+; CHECK-NEXT: v_sub_i32_e32 v4, vcc, v4, v8
+; CHECK-NEXT: v_subb_u32_e64 v7, s[4:5], v5, v6, vcc
+; CHECK-NEXT: v_sub_i32_e64 v5, s[4:5], v5, v6
; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v2
+; CHECK-NEXT: v_subb_u32_e32 v5, vcc, v5, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v2
+; CHECK-NEXT: v_sub_i32_e32 v4, vcc, v4, v2
+; CHECK-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v7, v9, v8, s[4:5]
-; CHECK-NEXT: v_add_i32_e64 v8, s[4:5], 2, v4
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v6, vcc
-; CHECK-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v5, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
-; CHECK-NEXT: v_add_i32_e64 v10, s[4:5], 1, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v5, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v7, v11, v9, s[4:5]
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, v10, v8, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: .LBB0_2: ; %Flow
-; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; CHECK-NEXT: s_cbranch_execz .LBB0_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v3, vcc, 0, v2
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_lo_u32 v3, v3, v1
-; CHECK-NEXT: v_mul_hi_u32 v3, v1, v3
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v3
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, 1, v0
+; CHECK-NEXT: v_cndmask_b32_e64 v6, v6, v8, s[4:5]
+; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v5, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v4, v2
+; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v7
+; CHECK-NEXT: v_addc_u32_e32 v4, vcc, 0, v8, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v8, v4, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT: s_cbranch_execz .LBB0_2
+; CHECK-NEXT: .LBB0_4:
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v6
+; CHECK-NEXT: v_sub_i32_e32 v1, vcc, 0, v2
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v1, v0
; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_mul_lo_u32 v3, v1, v2
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v1
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v0, v2
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v1
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v1, v3, vcc
-; CHECK-NEXT: .LBB0_4: ; %.split
-; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v0, v4
-; CHECK-NEXT: v_mov_b32_e32 v1, v5
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mul_lo_u32 v3, v0, v2
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, 1, v0
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
+; CHECK-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v0
+; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = udiv i64 %num, %den
ret i64 %result
@@ -632,288 +658,336 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-LABEL: v_udiv_v2i64:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_mov_b32_e32 v9, v1
-; CGP-NEXT: v_mov_b32_e32 v8, v0
-; CGP-NEXT: v_or_b32_e32 v0, v9, v5
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT: v_mov_b32_e32 v11, v1
+; CGP-NEXT: v_mov_b32_e32 v8, v2
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v4
+; CGP-NEXT: v_mov_b32_e32 v10, v0
+; CGP-NEXT: v_or_b32_e32 v1, v11, v5
+; CGP-NEXT: v_mov_b32_e32 v0, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT: v_mov_b32_e32 v9, v3
; CGP-NEXT: ; implicit-def: $vgpr0_vgpr1
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
; CGP-NEXT: s_cbranch_execz .LBB2_2
; CGP-NEXT: ; %bb.1:
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v5
-; CGP-NEXT: v_sub_i32_e32 v10, vcc, 0, v4
-; CGP-NEXT: v_subb_u32_e32 v11, vcc, 0, v5, vcc
-; CGP-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CGP-NEXT: v_rcp_f32_e32 v0, v0
+; CGP-NEXT: v_cvt_f32_u32_e32 v0, v5
+; CGP-NEXT: v_mac_f32_e32 v2, 0x4f800000, v0
+; CGP-NEXT: v_rcp_iflag_f32_e32 v0, v2
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, 0, v4
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, 0, v5, vcc
; CGP-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
; CGP-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; CGP-NEXT: v_trunc_f32_e32 v1, v1
-; CGP-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; CGP-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v1, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT: v_mul_lo_u32 v12, v10, v1
-; CGP-NEXT: v_mul_hi_u32 v13, v10, v0
-; CGP-NEXT: v_mul_lo_u32 v15, v11, v0
-; CGP-NEXT: v_mul_lo_u32 v14, v10, v0
+; CGP-NEXT: v_mul_lo_u32 v12, v2, v1
+; CGP-NEXT: v_mul_hi_u32 v13, v2, v0
+; CGP-NEXT: v_mul_lo_u32 v15, v3, v0
+; CGP-NEXT: v_mul_lo_u32 v14, v2, v0
; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v15
; CGP-NEXT: v_mul_hi_u32 v13, v0, v14
; CGP-NEXT: v_mul_lo_u32 v15, v0, v12
-; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
-; CGP-NEXT: v_mul_hi_u32 v17, v1, v12
-; CGP-NEXT: v_mul_lo_u32 v12, v1, v12
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; CGP-NEXT: v_mul_lo_u32 v16, v1, v14
; CGP-NEXT: v_mul_hi_u32 v14, v1, v14
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v16
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, v15, v14, vcc
-; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v17, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT: v_mul_lo_u32 v15, v1, v12
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT: v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc
+; CGP-NEXT: v_mul_hi_u32 v12, v2, v0
+; CGP-NEXT: v_mul_lo_u32 v13, v2, v1
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v0
+; CGP-NEXT: v_mul_lo_u32 v2, v2, v0
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v12, v3
+; CGP-NEXT: v_mul_hi_u32 v12, v0, v2
+; CGP-NEXT: v_mul_lo_u32 v13, v0, v3
+; CGP-NEXT: v_mul_lo_u32 v14, v1, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v1, v2
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v14, v0, v3
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_mul_lo_u32 v13, v1, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v14
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_mul_hi_u32 v3, v1, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; CGP-NEXT: v_mul_hi_u32 v2, v10, v0
+; CGP-NEXT: v_mul_lo_u32 v3, v10, v1
+; CGP-NEXT: v_mul_lo_u32 v12, v11, v0
+; CGP-NEXT: v_mul_hi_u32 v0, v11, v0
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v12, v10, v1
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v11, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v13, vcc
-; CGP-NEXT: v_mul_lo_u32 v12, v10, v0
-; CGP-NEXT: v_mul_lo_u32 v13, v10, v1
-; CGP-NEXT: v_mul_hi_u32 v10, v10, v0
-; CGP-NEXT: v_mul_lo_u32 v11, v11, v0
-; CGP-NEXT: v_mul_hi_u32 v14, v1, v12
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT: v_mul_lo_u32 v11, v0, v10
-; CGP-NEXT: v_mul_hi_u32 v13, v0, v12
-; CGP-NEXT: v_mul_hi_u32 v15, v0, v10
-; CGP-NEXT: v_mul_lo_u32 v12, v1, v12
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v15, vcc
-; CGP-NEXT: v_mul_hi_u32 v15, v1, v10
-; CGP-NEXT: v_mul_lo_u32 v10, v1, v10
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, v13, v14, vcc
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v15, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v10
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v11, vcc
-; CGP-NEXT: v_mul_lo_u32 v10, v8, v1
-; CGP-NEXT: v_mul_hi_u32 v11, v8, v0
-; CGP-NEXT: v_mul_hi_u32 v12, v8, v1
-; CGP-NEXT: v_mul_hi_u32 v13, v9, v1
-; CGP-NEXT: v_mul_lo_u32 v1, v9, v1
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
-; CGP-NEXT: v_mul_lo_u32 v12, v9, v0
-; CGP-NEXT: v_mul_hi_u32 v0, v9, v0
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v11, v0, vcc
-; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v13, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v10, vcc
-; CGP-NEXT: v_mul_lo_u32 v10, v4, v1
-; CGP-NEXT: v_mul_hi_u32 v11, v4, v0
-; CGP-NEXT: v_mul_lo_u32 v12, v5, v0
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT: v_mul_lo_u32 v11, v4, v0
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; CGP-NEXT: v_sub_i32_e32 v12, vcc, v9, v10
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, v8, v11
-; CGP-NEXT: v_subb_u32_e64 v11, s[4:5], v12, v5, vcc
-; CGP-NEXT: v_sub_i32_e64 v12, s[4:5], v8, v4
-; CGP-NEXT: v_subbrev_u32_e64 v11, s[4:5], 0, v11, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v5
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v4
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v12, v3
+; CGP-NEXT: v_mul_hi_u32 v1, v11, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v4, v0
+; CGP-NEXT: v_mul_lo_u32 v3, v4, v1
+; CGP-NEXT: v_mul_lo_u32 v13, v5, v0
+; CGP-NEXT: v_mul_lo_u32 v12, v4, v0
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v13
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v10, v12
+; CGP-NEXT: v_subb_u32_e64 v10, s[4:5], v11, v2, vcc
+; CGP-NEXT: v_sub_i32_e64 v2, s[4:5], v11, v2
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v5
+; CGP-NEXT: v_subb_u32_e32 v2, vcc, v2, v5, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v4
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v3, v4
; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v11, v5
-; CGP-NEXT: v_cndmask_b32_e64 v11, v13, v12, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v12, s[4:5], 2, v0
-; CGP-NEXT: v_subb_u32_e32 v9, vcc, v9, v10, vcc
-; CGP-NEXT: v_addc_u32_e64 v13, s[4:5], 0, v1, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v9, v5
-; CGP-NEXT: v_add_i32_e64 v14, s[4:5], 1, v0
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v8, v4
-; CGP-NEXT: v_addc_u32_e64 v15, s[4:5], 0, v1, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v9, v5
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v11
-; CGP-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v11, v15, v13, s[4:5]
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, v14, v12, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CGP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v10, v5
+; CGP-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v10, v11, v12, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v11, vcc, 1, v0
+; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v1, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v4
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v5
+; CGP-NEXT: v_cndmask_b32_e32 v2, v13, v3, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v11
+; CGP-NEXT: v_addc_u32_e32 v4, vcc, 0, v12, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; CGP-NEXT: v_cndmask_b32_e32 v2, v11, v3, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v12, v4, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; CGP-NEXT: ; implicit-def: $vgpr2
+; CGP-NEXT: ; implicit-def: $vgpr4
+; CGP-NEXT: ; implicit-def: $vgpr10
; CGP-NEXT: .LBB2_2: ; %Flow1
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
; CGP-NEXT: s_cbranch_execz .LBB2_4
; CGP-NEXT: ; %bb.3:
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT: v_rcp_iflag_f32_e32 v0, v2
; CGP-NEXT: v_sub_i32_e32 v1, vcc, 0, v4
-; CGP-NEXT: v_rcp_iflag_f32_e32 v0, v0
; CGP-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; CGP-NEXT: v_cvt_u32_f32_e32 v0, v0
; CGP-NEXT: v_mul_lo_u32 v1, v1, v0
; CGP-NEXT: v_mul_hi_u32 v1, v0, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_mul_hi_u32 v0, v8, v0
-; CGP-NEXT: v_mul_lo_u32 v1, v0, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v0
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v8, v1
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, v1, v4
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v0
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT: v_mul_hi_u32 v0, v10, v0
; CGP-NEXT: v_mov_b32_e32 v1, 0
+; CGP-NEXT: v_mul_lo_u32 v2, v0, v4
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v0
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v10, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v4
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v0
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; CGP-NEXT: .LBB2_4: ; %.split
-; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_or_b32_e32 v4, v3, v7
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT: v_cvt_f32_u32_e32 v4, v6
+; CGP-NEXT: v_or_b32_e32 v3, v9, v7
+; CGP-NEXT: v_mov_b32_e32 v2, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; CGP-NEXT: s_cbranch_execz .LBB2_6
-; CGP-NEXT: ; %bb.5:
-; CGP-NEXT: v_cvt_f32_u32_e32 v4, v6
-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v7
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, 0, v6
-; CGP-NEXT: v_subb_u32_e32 v9, vcc, 0, v7, vcc
-; CGP-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CGP-NEXT: v_rcp_f32_e32 v4, v4
-; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CGP-NEXT: v_trunc_f32_e32 v5, v5
-; CGP-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5
-; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4
-; CGP-NEXT: v_mul_lo_u32 v10, v8, v5
-; CGP-NEXT: v_mul_hi_u32 v11, v8, v4
-; CGP-NEXT: v_mul_lo_u32 v13, v9, v4
-; CGP-NEXT: v_mul_lo_u32 v12, v8, v4
+; CGP-NEXT: s_cbranch_execnz .LBB2_7
+; CGP-NEXT: ; %bb.5: ; %Flow
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT: s_cbranch_execnz .LBB2_8
+; CGP-NEXT: .LBB2_6: ; %.split.split
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT: s_setpc_b64 s[30:31]
+; CGP-NEXT: .LBB2_7:
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v7
+; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v2
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v4
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v6
+; CGP-NEXT: v_subb_u32_e32 v5, vcc, 0, v7, vcc
+; CGP-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CGP-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
+; CGP-NEXT: v_trunc_f32_e32 v3, v3
+; CGP-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3
+; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v10, v4, v3
+; CGP-NEXT: v_mul_hi_u32 v11, v4, v2
+; CGP-NEXT: v_mul_lo_u32 v13, v5, v2
+; CGP-NEXT: v_mul_lo_u32 v12, v4, v2
; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; CGP-NEXT: v_mul_hi_u32 v11, v4, v12
-; CGP-NEXT: v_mul_lo_u32 v13, v4, v10
-; CGP-NEXT: v_mul_hi_u32 v14, v4, v10
-; CGP-NEXT: v_mul_hi_u32 v15, v5, v10
-; CGP-NEXT: v_mul_lo_u32 v10, v5, v10
+; CGP-NEXT: v_mul_hi_u32 v11, v2, v12
+; CGP-NEXT: v_mul_lo_u32 v13, v2, v10
+; CGP-NEXT: v_mul_lo_u32 v14, v3, v12
+; CGP-NEXT: v_mul_hi_u32 v12, v3, v12
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
-; CGP-NEXT: v_mul_lo_u32 v14, v5, v12
-; CGP-NEXT: v_mul_hi_u32 v12, v5, v12
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, v13, v12, vcc
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v15, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v14, v2, v10
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_mul_lo_u32 v13, v3, v10
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_mul_hi_u32 v10, v3, v10
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v10, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v4, v2
+; CGP-NEXT: v_mul_lo_u32 v11, v4, v3
+; CGP-NEXT: v_mul_lo_u32 v5, v5, v2
+; CGP-NEXT: v_mul_lo_u32 v4, v4, v2
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT: v_mul_hi_u32 v10, v2, v4
+; CGP-NEXT: v_mul_lo_u32 v11, v2, v5
+; CGP-NEXT: v_mul_lo_u32 v12, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v12, v2, v5
; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; CGP-NEXT: v_mul_lo_u32 v11, v3, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_mul_hi_u32 v5, v3, v5
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v5, v11, vcc
-; CGP-NEXT: v_mul_lo_u32 v10, v8, v4
-; CGP-NEXT: v_mul_lo_u32 v11, v8, v5
-; CGP-NEXT: v_mul_hi_u32 v8, v8, v4
-; CGP-NEXT: v_mul_lo_u32 v9, v9, v4
-; CGP-NEXT: v_mul_hi_u32 v12, v5, v10
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT: v_mul_lo_u32 v9, v4, v8
-; CGP-NEXT: v_mul_hi_u32 v11, v4, v10
-; CGP-NEXT: v_mul_hi_u32 v13, v4, v8
-; CGP-NEXT: v_mul_lo_u32 v10, v5, v10
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
-; CGP-NEXT: v_mul_hi_u32 v13, v5, v8
-; CGP-NEXT: v_mul_lo_u32 v8, v5, v8
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT: v_addc_u32_e32 v9, vcc, v11, v12, vcc
-; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v13, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc
-; CGP-NEXT: v_mul_lo_u32 v8, v2, v5
-; CGP-NEXT: v_mul_hi_u32 v9, v2, v4
-; CGP-NEXT: v_mul_hi_u32 v10, v2, v5
-; CGP-NEXT: v_mul_hi_u32 v11, v3, v5
-; CGP-NEXT: v_mul_lo_u32 v5, v3, v5
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
-; CGP-NEXT: v_mul_lo_u32 v10, v3, v4
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v9, v4, vcc
-; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT: v_mul_hi_u32 v4, v8, v2
+; CGP-NEXT: v_mul_lo_u32 v5, v8, v3
+; CGP-NEXT: v_mul_lo_u32 v10, v9, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v9, v2
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
-; CGP-NEXT: v_mul_lo_u32 v8, v6, v5
-; CGP-NEXT: v_mul_hi_u32 v9, v6, v4
-; CGP-NEXT: v_mul_lo_u32 v10, v7, v4
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; CGP-NEXT: v_mul_lo_u32 v9, v6, v4
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT: v_sub_i32_e32 v10, vcc, v3, v8
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v9
-; CGP-NEXT: v_subb_u32_e64 v9, s[4:5], v10, v7, vcc
-; CGP-NEXT: v_sub_i32_e64 v10, s[4:5], v2, v6
-; CGP-NEXT: v_subbrev_u32_e64 v9, s[4:5], 0, v9, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v7
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v6
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v7
-; CGP-NEXT: v_cndmask_b32_e64 v9, v11, v10, s[4:5]
-; CGP-NEXT: v_add_i32_e64 v10, s[4:5], 2, v4
-; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
-; CGP-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v7
-; CGP-NEXT: v_add_i32_e64 v12, s[4:5], 1, v4
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_addc_u32_e64 v13, s[4:5], 0, v5, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v9
-; CGP-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v9, v13, v11, s[4:5]
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; CGP-NEXT: v_cndmask_b32_e64 v2, v12, v10, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; CGP-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CGP-NEXT: .LBB2_6: ; %Flow
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; CGP-NEXT: s_cbranch_execz .LBB2_8
-; CGP-NEXT: ; %bb.7:
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v6
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v6
-; CGP-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v4, v3
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v8, v3
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_mul_lo_u32 v5, v9, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT: v_mul_hi_u32 v3, v9, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v3, v6
-; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v3
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v6
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT: v_mul_hi_u32 v4, v6, v2
+; CGP-NEXT: v_mul_lo_u32 v5, v6, v3
+; CGP-NEXT: v_mul_lo_u32 v11, v7, v2
+; CGP-NEXT: v_mul_lo_u32 v10, v6, v2
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT: v_sub_i32_e32 v5, vcc, v8, v10
+; CGP-NEXT: v_subb_u32_e64 v8, s[4:5], v9, v4, vcc
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v9, v4
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v7
+; CGP-NEXT: v_subb_u32_e32 v4, vcc, v4, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v6
+; CGP-NEXT: v_sub_i32_e32 v5, vcc, v5, v6
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v8, v7
+; CGP-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v9, vcc, 1, v2
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v3, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v7
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v5, v6
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
+; CGP-NEXT: v_cndmask_b32_e32 v4, v11, v5, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v9
+; CGP-NEXT: v_addc_u32_e32 v6, vcc, 0, v10, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT: v_cndmask_b32_e32 v4, v9, v5, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v5, v10, v6, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v3
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v4, v3, v4, vcc
-; CGP-NEXT: v_mov_b32_e32 v5, 0
-; CGP-NEXT: .LBB2_8: ; %.split.split
-; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_mov_b32_e32 v2, v4
-; CGP-NEXT: v_mov_b32_e32 v3, v5
+; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT: ; implicit-def: $vgpr4
+; CGP-NEXT: ; implicit-def: $vgpr6
+; CGP-NEXT: ; implicit-def: $vgpr8
+; CGP-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT: s_cbranch_execz .LBB2_6
+; CGP-NEXT: .LBB2_8:
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v4
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v6
+; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v2, v8, v2
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_mul_lo_u32 v4, v2, v6
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v2
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v6
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
+; CGP-NEXT: v_sub_i32_e64 v5, s[4:5], v4, v6
+; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, 1, v2
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v4, v6
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
+; CGP-NEXT: s_or_b64 exec, exec, s[6:7]
; CGP-NEXT: s_setpc_b64 s[30:31]
%result = udiv <2 x i64> %num, %den
ret <2 x i64> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/unmerge-sgpr-s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/unmerge-sgpr-s16.mir
index 94a59eb1a92b2..d1cc6a30792f9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/unmerge-sgpr-s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/unmerge-sgpr-s16.mir
@@ -13,11 +13,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<2 x s16>) = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(p1) = COPY $sgpr2_sgpr3
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC]](s16)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[BUILD_VECTOR]](<2 x s16>)
; CHECK-NEXT: G_STORE [[COPY2]](<2 x s16>), [[COPY1]](p1) :: (store (<2 x s16>), addrspace 1)
@@ -43,15 +43,15 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(<4 x s16>) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(p1) = COPY $sgpr2_sgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:sgpr(<2 x s16>), [[UV1:%[0-9]+]]:sgpr(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:sgpr(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:sgpr(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:sgpr(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:sgpr(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:sgpr(s16) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC3]](s16)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[BUILD_VECTOR]](<2 x s16>)
; CHECK-NEXT: G_STORE [[COPY2]](<2 x s16>), [[COPY1]](p1) :: (store (<2 x s16>), addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 09ffbacea2c07..47544a4c11afb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,GISEL %s
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,CGP %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 -denormal-fp-math-f32=preserve-sign -mattr=+mad-mac-f32-insts < %s | FileCheck -check-prefixes=CHECK,CGP %s
; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
@@ -8,144 +8,170 @@ define i64 @v_urem_i64(i64 %num, i64 %den) {
; CHECK-LABEL: v_urem_i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_or_b32_e32 v4, v1, v3
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: v_mov_b32_e32 v5, v1
+; CHECK-NEXT: v_cvt_f32_u32_e32 v6, v2
+; CHECK-NEXT: v_mov_b32_e32 v4, v0
+; CHECK-NEXT: v_or_b32_e32 v1, v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CHECK-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB0_2
-; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v4, v2
-; CHECK-NEXT: v_cvt_f32_u32_e32 v5, v3
+; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_3
+; CHECK-NEXT: ; %bb.1: ; %Flow
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_4
+; CHECK-NEXT: .LBB0_2: ; %.split
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; CHECK-NEXT: .LBB0_3:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, v3
+; CHECK-NEXT: v_mac_f32_e32 v6, 0x4f800000, v0
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v6
; CHECK-NEXT: v_sub_i32_e32 v6, vcc, 0, v2
; CHECK-NEXT: v_subb_u32_e32 v7, vcc, 0, v3, vcc
-; CHECK-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CHECK-NEXT: v_rcp_f32_e32 v4, v4
-; CHECK-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CHECK-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CHECK-NEXT: v_trunc_f32_e32 v5, v5
-; CHECK-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CHECK-NEXT: v_cvt_u32_f32_e32 v5, v5
-; CHECK-NEXT: v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT: v_mul_lo_u32 v8, v6, v5
-; CHECK-NEXT: v_mul_hi_u32 v9, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v11, v7, v4
-; CHECK-NEXT: v_mul_lo_u32 v10, v6, v4
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; CHECK-NEXT: v_trunc_f32_e32 v1, v1
+; CHECK-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v8, v6, v1
+; CHECK-NEXT: v_mul_hi_u32 v9, v6, v0
+; CHECK-NEXT: v_mul_lo_u32 v11, v7, v0
+; CHECK-NEXT: v_mul_lo_u32 v10, v6, v0
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; CHECK-NEXT: v_mul_hi_u32 v9, v4, v10
-; CHECK-NEXT: v_mul_lo_u32 v11, v4, v8
-; CHECK-NEXT: v_mul_hi_u32 v12, v4, v8
-; CHECK-NEXT: v_mul_hi_u32 v13, v5, v8
-; CHECK-NEXT: v_mul_lo_u32 v8, v5, v8
+; CHECK-NEXT: v_mul_hi_u32 v9, v0, v10
+; CHECK-NEXT: v_mul_lo_u32 v11, v0, v8
+; CHECK-NEXT: v_mul_lo_u32 v12, v1, v10
+; CHECK-NEXT: v_mul_hi_u32 v10, v1, v10
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v11
-; CHECK-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
-; CHECK-NEXT: v_mul_lo_u32 v12, v5, v10
-; CHECK-NEXT: v_mul_hi_u32 v10, v5, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, v11, v10, vcc
-; CHECK-NEXT: v_addc_u32_e32 v10, vcc, 0, v13, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v12, v0, v8
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v11, v9
+; CHECK-NEXT: v_mul_lo_u32 v11, v1, v8
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CHECK-NEXT: v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc
+; CHECK-NEXT: v_mul_hi_u32 v8, v6, v0
+; CHECK-NEXT: v_mul_lo_u32 v9, v6, v1
+; CHECK-NEXT: v_mul_lo_u32 v7, v7, v0
+; CHECK-NEXT: v_mul_lo_u32 v6, v6, v0
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT: v_mul_hi_u32 v8, v0, v6
+; CHECK-NEXT: v_mul_lo_u32 v9, v0, v7
+; CHECK-NEXT: v_mul_lo_u32 v10, v1, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v1, v6
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v10, v0, v7
; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc
-; CHECK-NEXT: v_mul_lo_u32 v8, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v9, v6, v5
-; CHECK-NEXT: v_mul_hi_u32 v6, v6, v4
-; CHECK-NEXT: v_mul_lo_u32 v7, v7, v4
-; CHECK-NEXT: v_mul_hi_u32 v10, v5, v8
+; CHECK-NEXT: v_mul_lo_u32 v9, v1, v7
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v9
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT: v_mul_lo_u32 v7, v4, v6
-; CHECK-NEXT: v_mul_hi_u32 v9, v4, v8
-; CHECK-NEXT: v_mul_hi_u32 v11, v4, v6
-; CHECK-NEXT: v_mul_lo_u32 v8, v5, v8
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v9, v7
-; CHECK-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc
-; CHECK-NEXT: v_mul_hi_u32 v11, v5, v6
-; CHECK-NEXT: v_mul_lo_u32 v6, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT: v_mul_hi_u32 v7, v1, v7
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; CHECK-NEXT: v_addc_u32_e32 v7, vcc, v9, v10, vcc
-; CHECK-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
+; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, v0
+; CHECK-NEXT: v_mul_lo_u32 v7, v4, v1
+; CHECK-NEXT: v_mul_lo_u32 v8, v5, v0
+; CHECK-NEXT: v_mul_hi_u32 v0, v5, v0
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v8, v4, v1
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v6
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc
-; CHECK-NEXT: v_mul_lo_u32 v6, v0, v5
-; CHECK-NEXT: v_mul_hi_u32 v7, v0, v4
-; CHECK-NEXT: v_mul_hi_u32 v8, v0, v5
-; CHECK-NEXT: v_mul_hi_u32 v9, v1, v5
-; CHECK-NEXT: v_mul_lo_u32 v5, v1, v5
+; CHECK-NEXT: v_mul_lo_u32 v7, v5, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v7
+; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT: v_mul_hi_u32 v1, v5, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; CHECK-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; CHECK-NEXT: v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; CHECK-NEXT: v_mul_lo_u32 v8, v1, v4
-; CHECK-NEXT: v_mul_hi_u32 v4, v1, v4
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT: v_addc_u32_e32 v4, vcc, v7, v4, vcc
-; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v9, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; CHECK-NEXT: v_mul_lo_u32 v5, v2, v5
-; CHECK-NEXT: v_mul_hi_u32 v6, v2, v4
-; CHECK-NEXT: v_mul_lo_u32 v7, v3, v4
-; CHECK-NEXT: v_mul_lo_u32 v4, v2, v4
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; CHECK-NEXT: v_sub_i32_e32 v6, vcc, v1, v5
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; CHECK-NEXT: v_subb_u32_e64 v4, s[4:5], v6, v3, vcc
-; CHECK-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v2
-; CHECK-NEXT: v_subbrev_u32_e64 v7, s[6:7], 0, v4, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[6:7], v7, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7]
-; CHECK-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v2
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[6:7], v7, v3
-; CHECK-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v3, s[4:5]
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[6:7]
-; CHECK-NEXT: v_sub_i32_e64 v9, s[4:5], v6, v2
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
-; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v8
-; CHECK-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v5, v1, v4, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v1, v6, v9, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: .LBB0_2: ; %Flow
-; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; CHECK-NEXT: s_cbranch_execz .LBB0_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v3, vcc, 0, v2
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_lo_u32 v3, v3, v1
-; CHECK-NEXT: v_mul_hi_u32 v3, v1, v3
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v3
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v6
+; CHECK-NEXT: v_mul_hi_u32 v6, v2, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v2, v1
+; CHECK-NEXT: v_mul_lo_u32 v7, v2, v0
+; CHECK-NEXT: v_mul_lo_u32 v0, v3, v0
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v6, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v4, v7
+; CHECK-NEXT: v_subb_u32_e64 v4, s[4:5], v5, v0, vcc
+; CHECK-NEXT: v_sub_i32_e64 v0, s[4:5], v5, v0
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v2
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v3
+; CHECK-NEXT: v_subb_u32_e32 v0, vcc, v0, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[4:5]
+; CHECK-NEXT: v_sub_i32_e32 v6, vcc, v1, v2
+; CHECK-NEXT: v_subbrev_u32_e64 v7, s[4:5], 0, v0, vcc
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v3
+; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v2
+; CHECK-NEXT: v_subb_u32_e32 v0, vcc, v0, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v3
+; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v6, v2
+; CHECK-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[4:5]
+; CHECK-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
+; CHECK-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v3, v7, v0, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT: s_cbranch_execz .LBB0_2
+; CHECK-NEXT: .LBB0_4:
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v6
+; CHECK-NEXT: v_sub_i32_e32 v1, vcc, 0, v2
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_lo_u32 v1, v1, v0
; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_mul_lo_u32 v1, v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_mul_hi_u32 v0, v4, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mul_lo_u32 v0, v0, v2
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v4, v0
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v0, v2
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT: v_sub_i32_e32 v3, vcc, v0, v2
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
-; CHECK-NEXT: .LBB0_4: ; %.split
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mov_b32_e32 v0, v4
-; CHECK-NEXT: v_mov_b32_e32 v1, v5
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = urem i64 %num, %den
ret i64 %result
@@ -626,282 +652,330 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
; CGP-LABEL: v_urem_v2i64:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_mov_b32_e32 v9, v1
-; CGP-NEXT: v_mov_b32_e32 v8, v0
-; CGP-NEXT: v_or_b32_e32 v0, v9, v5
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT: v_mov_b32_e32 v11, v1
+; CGP-NEXT: v_mov_b32_e32 v8, v2
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v4
+; CGP-NEXT: v_mov_b32_e32 v10, v0
+; CGP-NEXT: v_or_b32_e32 v1, v11, v5
+; CGP-NEXT: v_mov_b32_e32 v0, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT: v_mov_b32_e32 v9, v3
; CGP-NEXT: ; implicit-def: $vgpr0_vgpr1
; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CGP-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
; CGP-NEXT: s_cbranch_execz .LBB2_2
; CGP-NEXT: ; %bb.1:
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v5
-; CGP-NEXT: v_sub_i32_e32 v10, vcc, 0, v4
-; CGP-NEXT: v_subb_u32_e32 v11, vcc, 0, v5, vcc
-; CGP-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CGP-NEXT: v_rcp_f32_e32 v0, v0
+; CGP-NEXT: v_cvt_f32_u32_e32 v0, v5
+; CGP-NEXT: v_mac_f32_e32 v2, 0x4f800000, v0
+; CGP-NEXT: v_rcp_iflag_f32_e32 v0, v2
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, 0, v4
+; CGP-NEXT: v_subb_u32_e32 v3, vcc, 0, v5, vcc
; CGP-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
; CGP-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; CGP-NEXT: v_trunc_f32_e32 v1, v1
-; CGP-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; CGP-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v1, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT: v_mul_lo_u32 v12, v10, v1
-; CGP-NEXT: v_mul_hi_u32 v13, v10, v0
-; CGP-NEXT: v_mul_lo_u32 v15, v11, v0
-; CGP-NEXT: v_mul_lo_u32 v14, v10, v0
+; CGP-NEXT: v_mul_lo_u32 v12, v2, v1
+; CGP-NEXT: v_mul_hi_u32 v13, v2, v0
+; CGP-NEXT: v_mul_lo_u32 v15, v3, v0
+; CGP-NEXT: v_mul_lo_u32 v14, v2, v0
; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v15
; CGP-NEXT: v_mul_hi_u32 v13, v0, v14
; CGP-NEXT: v_mul_lo_u32 v15, v0, v12
-; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
-; CGP-NEXT: v_mul_hi_u32 v17, v1, v12
-; CGP-NEXT: v_mul_lo_u32 v12, v1, v12
-; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
-; CGP-NEXT: v_addc_u32_e32 v15, vcc, 0, v16, vcc
; CGP-NEXT: v_mul_lo_u32 v16, v1, v14
; CGP-NEXT: v_mul_hi_u32 v14, v1, v14
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v13, vcc, v13, v16
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, v15, v14, vcc
-; CGP-NEXT: v_addc_u32_e32 v14, vcc, 0, v17, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v16, v0, v12
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT: v_mul_lo_u32 v15, v1, v12
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT: v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc
+; CGP-NEXT: v_mul_hi_u32 v12, v2, v0
+; CGP-NEXT: v_mul_lo_u32 v13, v2, v1
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v0
+; CGP-NEXT: v_mul_lo_u32 v2, v2, v0
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v12, v3
+; CGP-NEXT: v_mul_hi_u32 v12, v0, v2
+; CGP-NEXT: v_mul_lo_u32 v13, v0, v3
+; CGP-NEXT: v_mul_lo_u32 v14, v1, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v1, v2
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v14, v0, v3
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_mul_lo_u32 v13, v1, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v14
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_mul_hi_u32 v3, v1, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v12
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; CGP-NEXT: v_mul_hi_u32 v2, v10, v0
+; CGP-NEXT: v_mul_lo_u32 v3, v10, v1
+; CGP-NEXT: v_mul_lo_u32 v12, v11, v0
+; CGP-NEXT: v_mul_hi_u32 v0, v11, v0
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v12
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v12, v10, v1
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v11, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v13, vcc
-; CGP-NEXT: v_mul_lo_u32 v12, v10, v0
-; CGP-NEXT: v_mul_lo_u32 v13, v10, v1
-; CGP-NEXT: v_mul_hi_u32 v10, v10, v0
-; CGP-NEXT: v_mul_lo_u32 v11, v11, v0
-; CGP-NEXT: v_mul_hi_u32 v14, v1, v12
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT: v_mul_lo_u32 v11, v0, v10
-; CGP-NEXT: v_mul_hi_u32 v13, v0, v12
-; CGP-NEXT: v_mul_hi_u32 v15, v0, v10
-; CGP-NEXT: v_mul_lo_u32 v12, v1, v12
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v15, vcc
-; CGP-NEXT: v_mul_hi_u32 v15, v1, v10
-; CGP-NEXT: v_mul_lo_u32 v10, v1, v10
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, v13, v14, vcc
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v15, vcc
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v10
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v11, vcc
-; CGP-NEXT: v_mul_lo_u32 v10, v8, v1
-; CGP-NEXT: v_mul_hi_u32 v11, v8, v0
-; CGP-NEXT: v_mul_hi_u32 v12, v8, v1
-; CGP-NEXT: v_mul_hi_u32 v13, v9, v1
-; CGP-NEXT: v_mul_lo_u32 v1, v9, v1
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
-; CGP-NEXT: v_mul_lo_u32 v12, v9, v0
-; CGP-NEXT: v_mul_hi_u32 v0, v9, v0
-; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
-; CGP-NEXT: v_addc_u32_e32 v0, vcc, v11, v0, vcc
-; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v13, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v10, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v12, v3
+; CGP-NEXT: v_mul_hi_u32 v1, v11, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v4, v0
; CGP-NEXT: v_mul_lo_u32 v1, v4, v1
-; CGP-NEXT: v_mul_hi_u32 v10, v4, v0
-; CGP-NEXT: v_mul_lo_u32 v11, v5, v0
-; CGP-NEXT: v_mul_lo_u32 v0, v4, v0
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v10, v1
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v11
-; CGP-NEXT: v_sub_i32_e32 v10, vcc, v9, v1
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v8, v0
-; CGP-NEXT: v_subb_u32_e64 v8, s[4:5], v10, v5, vcc
-; CGP-NEXT: v_sub_i32_e64 v10, s[4:5], v0, v4
-; CGP-NEXT: v_subbrev_u32_e64 v11, s[6:7], 0, v8, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v11, v5
-; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v10, v4
-; CGP-NEXT: v_subb_u32_e32 v1, vcc, v9, v1, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[6:7], v11, v5
-; CGP-NEXT: v_subb_u32_e64 v8, s[4:5], v8, v5, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5
-; CGP-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[6:7]
-; CGP-NEXT: v_sub_i32_e64 v13, s[4:5], v10, v4
-; CGP-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_subbrev_u32_e64 v8, s[4:5], 0, v8, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v12
-; CGP-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v8, v11, v8, s[4:5]
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: v_cndmask_b32_e64 v4, v10, v13, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CGP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CGP-NEXT: v_mul_lo_u32 v3, v4, v0
+; CGP-NEXT: v_mul_lo_u32 v0, v5, v0
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v2, v1
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CGP-NEXT: v_sub_i32_e32 v1, vcc, v10, v3
+; CGP-NEXT: v_subb_u32_e64 v2, s[4:5], v11, v0, vcc
+; CGP-NEXT: v_sub_i32_e64 v0, s[4:5], v11, v0
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v5
+; CGP-NEXT: v_cndmask_b32_e64 v3, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v4
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v2, v5
+; CGP-NEXT: v_subb_u32_e32 v0, vcc, v0, v5, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v3, v3, v10, s[4:5]
+; CGP-NEXT: v_sub_i32_e32 v10, vcc, v1, v4
+; CGP-NEXT: v_subbrev_u32_e64 v11, s[4:5], 0, v0, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v5
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v4
+; CGP-NEXT: v_subb_u32_e32 v0, vcc, v0, v5, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v11, v5
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v10, v4
+; CGP-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[4:5]
+; CGP-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
+; CGP-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v5, v11, v0, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; CGP-NEXT: ; implicit-def: $vgpr2
+; CGP-NEXT: ; implicit-def: $vgpr4
+; CGP-NEXT: ; implicit-def: $vgpr10
; CGP-NEXT: .LBB2_2: ; %Flow1
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
; CGP-NEXT: s_cbranch_execz .LBB2_4
; CGP-NEXT: ; %bb.3:
-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT: v_rcp_iflag_f32_e32 v0, v2
; CGP-NEXT: v_sub_i32_e32 v1, vcc, 0, v4
-; CGP-NEXT: v_rcp_iflag_f32_e32 v0, v0
; CGP-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; CGP-NEXT: v_cvt_u32_f32_e32 v0, v0
; CGP-NEXT: v_mul_lo_u32 v1, v1, v0
; CGP-NEXT: v_mul_hi_u32 v1, v0, v1
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_mul_hi_u32 v0, v8, v0
+; CGP-NEXT: v_mul_hi_u32 v0, v10, v0
+; CGP-NEXT: v_mov_b32_e32 v1, 0
; CGP-NEXT: v_mul_lo_u32 v0, v0, v4
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v8, v0
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v10, v0
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v4
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v4
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CGP-NEXT: v_mov_b32_e32 v1, 0
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; CGP-NEXT: .LBB2_4: ; %.split
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_or_b32_e32 v4, v3, v7
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; CGP-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CGP-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; CGP-NEXT: s_cbranch_execz .LBB2_6
-; CGP-NEXT: ; %bb.5:
; CGP-NEXT: v_cvt_f32_u32_e32 v4, v6
-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v7
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, 0, v6
-; CGP-NEXT: v_subb_u32_e32 v9, vcc, 0, v7, vcc
-; CGP-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; CGP-NEXT: v_rcp_f32_e32 v4, v4
-; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; CGP-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; CGP-NEXT: v_trunc_f32_e32 v5, v5
-; CGP-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5
-; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4
-; CGP-NEXT: v_mul_lo_u32 v10, v8, v5
-; CGP-NEXT: v_mul_hi_u32 v11, v8, v4
-; CGP-NEXT: v_mul_lo_u32 v13, v9, v4
-; CGP-NEXT: v_mul_lo_u32 v12, v8, v4
+; CGP-NEXT: v_or_b32_e32 v3, v9, v7
+; CGP-NEXT: v_mov_b32_e32 v2, 0
+; CGP-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CGP-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; CGP-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
+; CGP-NEXT: s_cbranch_execnz .LBB2_7
+; CGP-NEXT: ; %bb.5: ; %Flow
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_cbranch_execnz .LBB2_8
+; CGP-NEXT: .LBB2_6: ; %.split.split
+; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
+; CGP-NEXT: s_setpc_b64 s[30:31]
+; CGP-NEXT: .LBB2_7:
+; CGP-NEXT: v_cvt_f32_u32_e32 v2, v7
+; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v2
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v4
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v6
+; CGP-NEXT: v_subb_u32_e32 v5, vcc, 0, v7, vcc
+; CGP-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CGP-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
+; CGP-NEXT: v_trunc_f32_e32 v3, v3
+; CGP-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3
+; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v10, v4, v3
+; CGP-NEXT: v_mul_hi_u32 v11, v4, v2
+; CGP-NEXT: v_mul_lo_u32 v13, v5, v2
+; CGP-NEXT: v_mul_lo_u32 v12, v4, v2
; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v13
-; CGP-NEXT: v_mul_hi_u32 v11, v4, v12
-; CGP-NEXT: v_mul_lo_u32 v13, v4, v10
-; CGP-NEXT: v_mul_hi_u32 v14, v4, v10
-; CGP-NEXT: v_mul_hi_u32 v15, v5, v10
-; CGP-NEXT: v_mul_lo_u32 v10, v5, v10
+; CGP-NEXT: v_mul_hi_u32 v11, v2, v12
+; CGP-NEXT: v_mul_lo_u32 v13, v2, v10
+; CGP-NEXT: v_mul_lo_u32 v14, v3, v12
+; CGP-NEXT: v_mul_hi_u32 v12, v3, v12
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; CGP-NEXT: v_addc_u32_e32 v13, vcc, 0, v14, vcc
-; CGP-NEXT: v_mul_lo_u32 v14, v5, v12
-; CGP-NEXT: v_mul_hi_u32 v12, v5, v12
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v14
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, v13, v12, vcc
-; CGP-NEXT: v_addc_u32_e32 v12, vcc, 0, v15, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v14, v2, v10
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT: v_mul_lo_u32 v13, v3, v10
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT: v_mul_hi_u32 v10, v3, v10
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v10, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v4, v2
+; CGP-NEXT: v_mul_lo_u32 v11, v4, v3
+; CGP-NEXT: v_mul_lo_u32 v5, v5, v2
+; CGP-NEXT: v_mul_lo_u32 v4, v4, v2
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT: v_mul_hi_u32 v10, v2, v4
+; CGP-NEXT: v_mul_lo_u32 v11, v2, v5
+; CGP-NEXT: v_mul_lo_u32 v12, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v12, v2, v5
; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; CGP-NEXT: v_mul_lo_u32 v11, v3, v5
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT: v_mul_hi_u32 v5, v3, v5
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v5, v11, vcc
-; CGP-NEXT: v_mul_lo_u32 v10, v8, v4
-; CGP-NEXT: v_mul_lo_u32 v11, v8, v5
-; CGP-NEXT: v_mul_hi_u32 v8, v8, v4
-; CGP-NEXT: v_mul_lo_u32 v9, v9, v4
-; CGP-NEXT: v_mul_hi_u32 v12, v5, v10
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT: v_mul_lo_u32 v9, v4, v8
-; CGP-NEXT: v_mul_hi_u32 v11, v4, v10
-; CGP-NEXT: v_mul_hi_u32 v13, v4, v8
-; CGP-NEXT: v_mul_lo_u32 v10, v5, v10
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; CGP-NEXT: v_addc_u32_e32 v11, vcc, 0, v13, vcc
-; CGP-NEXT: v_mul_hi_u32 v13, v5, v8
-; CGP-NEXT: v_mul_lo_u32 v8, v5, v8
-; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT: v_addc_u32_e32 v9, vcc, v11, v12, vcc
-; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v13, vcc
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
-; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc
-; CGP-NEXT: v_mul_lo_u32 v8, v2, v5
-; CGP-NEXT: v_mul_hi_u32 v9, v2, v4
-; CGP-NEXT: v_mul_hi_u32 v10, v2, v5
-; CGP-NEXT: v_mul_hi_u32 v11, v3, v5
-; CGP-NEXT: v_mul_lo_u32 v5, v3, v5
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc
-; CGP-NEXT: v_mul_lo_u32 v10, v3, v4
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT: v_addc_u32_e32 v4, vcc, v9, v4, vcc
-; CGP-NEXT: v_addc_u32_e32 v8, vcc, 0, v11, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v10
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT: v_mul_hi_u32 v4, v8, v2
+; CGP-NEXT: v_mul_lo_u32 v5, v8, v3
+; CGP-NEXT: v_mul_lo_u32 v10, v9, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v9, v2
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; CGP-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
-; CGP-NEXT: v_mul_lo_u32 v5, v6, v5
-; CGP-NEXT: v_mul_hi_u32 v8, v6, v4
-; CGP-NEXT: v_mul_lo_u32 v9, v7, v4
-; CGP-NEXT: v_mul_lo_u32 v4, v6, v4
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
-; CGP-NEXT: v_sub_i32_e32 v8, vcc, v3, v5
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v8, v7, vcc
-; CGP-NEXT: v_sub_i32_e64 v8, s[4:5], v2, v6
-; CGP-NEXT: v_subbrev_u32_e64 v9, s[6:7], 0, v4, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v9, v7
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_ge_u32_e64 s[6:7], v8, v6
-; CGP-NEXT: v_subb_u32_e32 v3, vcc, v3, v5, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[6:7]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[6:7], v9, v7
-; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v7, s[4:5]
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v7
-; CGP-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[6:7]
-; CGP-NEXT: v_sub_i32_e64 v11, s[4:5], v8, v6
-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
-; CGP-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v10
-; CGP-NEXT: v_cndmask_b32_e32 v5, v5, v6, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v4, v9, v4, s[4:5]
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; CGP-NEXT: v_cndmask_b32_e32 v5, v3, v4, vcc
-; CGP-NEXT: v_cndmask_b32_e64 v3, v8, v11, s[4:5]
-; CGP-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc
-; CGP-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CGP-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CGP-NEXT: .LBB2_6: ; %Flow
-; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; CGP-NEXT: s_cbranch_execz .LBB2_8
-; CGP-NEXT: ; %bb.7:
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v6
-; CGP-NEXT: v_sub_i32_e32 v4, vcc, 0, v6
-; CGP-NEXT: v_mov_b32_e32 v5, 0
-; CGP-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT: v_mul_lo_u32 v4, v4, v3
-; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v10, v8, v3
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_mul_lo_u32 v5, v9, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT: v_mul_hi_u32 v3, v9, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v5, v4
; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v4, v6, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v6, v3
+; CGP-NEXT: v_mul_lo_u32 v5, v6, v2
+; CGP-NEXT: v_mul_lo_u32 v2, v7, v2
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v8, v5
+; CGP-NEXT: v_subb_u32_e64 v4, s[4:5], v9, v2, vcc
+; CGP-NEXT: v_sub_i32_e64 v2, s[4:5], v9, v2
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v7
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v6
+; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v7
+; CGP-NEXT: v_subb_u32_e32 v2, vcc, v2, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[4:5]
+; CGP-NEXT: v_sub_i32_e32 v8, vcc, v3, v6
+; CGP-NEXT: v_subbrev_u32_e64 v9, s[4:5], 0, v2, vcc
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v7
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v6
+; CGP-NEXT: v_subb_u32_e32 v2, vcc, v2, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v7
+; CGP-NEXT: v_sub_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; CGP-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v7, v9, v2, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc
+; CGP-NEXT: ; implicit-def: $vgpr4
+; CGP-NEXT: ; implicit-def: $vgpr6
+; CGP-NEXT: ; implicit-def: $vgpr8
+; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT: s_cbranch_execz .LBB2_6
+; CGP-NEXT: .LBB2_8:
+; CGP-NEXT: v_rcp_iflag_f32_e32 v2, v4
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v6
+; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
-; CGP-NEXT: v_mul_lo_u32 v3, v3, v6
-; CGP-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT: v_mul_hi_u32 v2, v8, v2
+; CGP-NEXT: v_mov_b32_e32 v3, 0
+; CGP-NEXT: v_mul_lo_u32 v2, v2, v6
+; CGP-NEXT: v_sub_i32_e32 v2, vcc, v8, v2
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v6
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v2, v6
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
-; CGP-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc
-; CGP-NEXT: .LBB2_8: ; %.split.split
+; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
-; CGP-NEXT: v_mov_b32_e32 v2, v4
-; CGP-NEXT: v_mov_b32_e32 v3, v5
; CGP-NEXT: s_setpc_b64 s[30:31]
%result = urem <2 x i64> %num, %den
ret <2 x i64> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index f24e6327cf021..14854738c4f9d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; xUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
@@ -77,27 +77,55 @@ define amdgpu_ps i7 @s_usubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
;
; GFX8-LABEL: s_usubsat_i7:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX8-NEXT: s_and_b32 s0, s0, 0x7f
-; GFX8-NEXT: s_max_u32 s0, s0, s1
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 9
+; GFX8-NEXT: s_lshl_b32 s0, s0, 9
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 9
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_i7:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX9-NEXT: s_and_b32 s0, s0, 0x7f
-; GFX9-NEXT: s_max_u32 s0, s0, s1
-; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 9
+; GFX9-NEXT: s_lshl_b32 s0, s0, 9
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 9
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_usubsat_i7:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0x7f
-; GFX10PLUS-NEXT: s_max_u32 s0, s0, s1
-; GFX10PLUS-NEXT: s_sub_i32 s0, s0, s1
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_usubsat_i7:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b32 s0, s0, 9
+; GFX10-NEXT: s_lshl_b32 s1, s1, 9
+; GFX10-NEXT: v_sub_nc_u16 v0, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_lshr_b32 s0, s0, 9
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_usubsat_i7:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 9
+; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.l, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 9
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_usubsat_i7:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 9
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 9
+; GFX11-FAKE16-NEXT: v_sub_nc_u16 v0, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 9
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i7 @llvm.usub.sat.i7(i7 %lhs, i7 %rhs)
ret i7 %result
}
@@ -173,27 +201,55 @@ define amdgpu_ps i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
;
; GFX8-LABEL: s_usubsat_i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s1, 0xff
-; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_max_u32 s0, s0, s1
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_max_u32 s0, s0, s1
-; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_usubsat_i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_max_u32 s0, s0, s1
-; GFX10PLUS-NEXT: s_sub_i32 s0, s0, s1
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_usubsat_i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b32 s0, s0, 8
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10-NEXT: v_sub_nc_u16 v0, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_usubsat_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.l, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_usubsat_i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-FAKE16-NEXT: v_sub_nc_u16 v0, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
ret i8 %result
}
@@ -307,16 +363,22 @@ define amdgpu_ps i16 @s_usubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX8-LABEL: s_usubsat_v2i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX8-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX8-NEXT: s_and_b32 s1, s1, 0xff
-; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_lshr_b32 s2, s2, 8
-; GFX8-NEXT: s_lshr_b32 s3, s3, 8
-; GFX8-NEXT: s_max_u32 s0, s0, s1
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: s_max_u32 s1, s3, s2
-; GFX8-NEXT: s_sub_i32 s1, s1, s2
+; GFX8-NEXT: s_lshr_b32 s3, s1, 8
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_sub_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s1, s1, 8
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
; GFX8-NEXT: s_lshl_b32 s1, s1, 8
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
@@ -337,21 +399,63 @@ define amdgpu_ps i16 @s_usubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_usubsat_v2i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX10PLUS-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX10PLUS-NEXT: s_lshr_b32 s2, s2, 8
-; GFX10PLUS-NEXT: s_lshr_b32 s3, s3, 8
-; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_max_u32 s3, s3, s2
-; GFX10PLUS-NEXT: s_max_u32 s0, s0, s1
-; GFX10PLUS-NEXT: s_sub_i32 s2, s3, s2
-; GFX10PLUS-NEXT: s_sub_i32 s0, s0, s1
-; GFX10PLUS-NEXT: s_lshl_b32 s1, s2, 8
-; GFX10PLUS-NEXT: s_or_b32 s0, s0, s1
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_usubsat_v2i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s2, 0xffff, s1
+; GFX10-NEXT: s_and_b32 s3, 0xffff, s0
+; GFX10-NEXT: s_lshr_b32 s2, s2, 8
+; GFX10-NEXT: s_lshr_b32 s3, s3, 8
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_max_u32 s3, s3, s2
+; GFX10-NEXT: s_max_u32 s0, s0, s1
+; GFX10-NEXT: s_sub_i32 s2, s3, s2
+; GFX10-NEXT: s_sub_i32 s0, s0, s1
+; GFX10-NEXT: s_lshl_b32 s1, s2, 8
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_usubsat_v2i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-TRUE16-NEXT: v_pk_sub_u16 v0, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_usubsat_v2i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s2, 0xffff, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s3, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_max_u32 s3, s3, s2
+; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_sub_i32 s2, s3, s2
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s2, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%result = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
@@ -1559,27 +1663,35 @@ define amdgpu_ps i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
;
; GFX8-LABEL: s_usubsat_i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX8-NEXT: s_max_u32 s0, s0, s1
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX9-NEXT: s_max_u32 s0, s0, s1
-; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_usubsat_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX10PLUS-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX10PLUS-NEXT: s_max_u32 s0, s0, s1
-; GFX10PLUS-NEXT: s_sub_i32 s0, s0, s1
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_usubsat_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_sub_nc_u16 v0, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_usubsat_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.l, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_usubsat_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_sub_nc_u16 v0, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
ret i16 %result
}
@@ -1836,26 +1948,30 @@ define <2 x float> @v_usubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
; GFX6-LABEL: v_usubsat_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_max_u32_e32 v0, v0, v2
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT: v_max_u32_e32 v2, v7, v6
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_max_u32_e32 v2, v5, v4
-; GFX6-NEXT: v_max_u32_e32 v1, v1, v3
+; GFX6-NEXT: v_min_u32_e32 v2, v0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_min_u32_e32 v4, v2, v4
+; GFX6-NEXT: v_min_u32_e32 v3, v1, v3
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_min_u32_e32 v4, v3, v4
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX6-NEXT: v_alignbit_b32 v1, v3, v1, 16
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubsat_v4i16:
@@ -1981,36 +2097,42 @@ define <3 x float> @v_usubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
; GFX6-LABEL: v_usubsat_v6i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_max_u32_e32 v0, v0, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; GFX6-NEXT: v_max_u32_e32 v3, v11, v10
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v10
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX6-NEXT: v_max_u32_e32 v3, v9, v8
-; GFX6-NEXT: v_max_u32_e32 v1, v1, v4
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v8
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX6-NEXT: v_max_u32_e32 v3, v7, v6
-; GFX6-NEXT: v_max_u32_e32 v2, v2, v5
+; GFX6-NEXT: v_min_u32_e32 v3, v0, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_min_u32_e32 v6, v3, v6
+; GFX6-NEXT: v_min_u32_e32 v4, v1, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v5
; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v6
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_min_u32_e32 v6, v4, v6
+; GFX6-NEXT: v_min_u32_e32 v5, v2, v5
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v6
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX6-NEXT: v_min_u32_e32 v6, v5, v6
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX6-NEXT: v_alignbit_b32 v1, v4, v1, 16
+; GFX6-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubsat_v6i16:
@@ -2159,46 +2281,54 @@ define <4 x float> @v_usubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
; GFX6-LABEL: v_usubsat_v8i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v14, 16, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_max_u32_e32 v0, v0, v4
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; GFX6-NEXT: v_max_u32_e32 v4, v15, v14
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v14
-; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX6-NEXT: v_max_u32_e32 v4, v13, v12
-; GFX6-NEXT: v_max_u32_e32 v1, v1, v5
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v12
-; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v2
+; GFX6-NEXT: v_min_u32_e32 v4, v0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_min_u32_e32 v8, v4, v8
+; GFX6-NEXT: v_min_u32_e32 v5, v1, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v8
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX6-NEXT: v_max_u32_e32 v4, v11, v10
-; GFX6-NEXT: v_max_u32_e32 v2, v2, v6
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v10
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v9
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_min_u32_e32 v8, v5, v8
+; GFX6-NEXT: v_min_u32_e32 v6, v2, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v5, v8
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_max_u32_e32 v4, v9, v8
-; GFX6-NEXT: v_max_u32_e32 v3, v3, v7
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX6-NEXT: v_min_u32_e32 v8, v6, v8
+; GFX6-NEXT: v_min_u32_e32 v7, v3, v7
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v6, v8
; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v7
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v11
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v15
+; GFX6-NEXT: v_min_u32_e32 v8, v7, v8
+; GFX6-NEXT: v_sub_i32_e32 v7, vcc, v7, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX6-NEXT: v_alignbit_b32 v1, v5, v1, 16
+; GFX6-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX6-NEXT: v_alignbit_b32 v3, v7, v3, 16
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubsat_v8i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
index ea3aadb564380..d5be96a7dee75 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX900 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck -check-prefixes=GCN,GFX906 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX900 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck -check-prefixes=GCN,GFX906 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10 %s
; FIXME: regbankcombiner regression, related to:
; - looking through copy and splitting G_CONSTANT i64 to two i32 constants
@@ -209,39 +209,14 @@ entry:
}
define i64 @vector_xnor_i64_one_use(i64 %a, i64 %b) {
-; GFX7-LABEL: vector_xnor_i64_one_use:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_not_b32_e32 v1, v1
-; GFX7-NEXT: v_not_b32_e32 v0, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: vector_xnor_i64_one_use:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_not_b32_e32 v1, v1
-; GFX8-NEXT: v_not_b32_e32 v0, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX900-LABEL: vector_xnor_i64_one_use:
-; GFX900: ; %bb.0: ; %entry
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX900-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX900-NEXT: v_not_b32_e32 v1, v1
-; GFX900-NEXT: v_not_b32_e32 v0, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX906-LABEL: vector_xnor_i64_one_use:
-; GFX906: ; %bb.0: ; %entry
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_xnor_b32_e32 v1, v1, v3
-; GFX906-NEXT: v_xnor_b32_e32 v0, v0, v2
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; GCN-LABEL: vector_xnor_i64_one_use:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_xor_b32_e32 v0, v0, v2
+; GCN-NEXT: v_xor_b32_e32 v1, v1, v3
+; GCN-NEXT: v_xor_b32_e32 v0, -1, v0
+; GCN-NEXT: v_xor_b32_e32 v1, -1, v1
+; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: vector_xnor_i64_one_use:
; GFX10: ; %bb.0: ; %entry
@@ -327,41 +302,44 @@ define amdgpu_ps <2 x float> @xnor_i64_s_v_one_use(i64 inreg %a, i64 %b64) {
; GFX7-LABEL: xnor_i64_s_v_one_use:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 29
-; GFX7-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX7-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX7-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX7-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX7-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX7-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: xnor_i64_s_v_one_use:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX8-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX8-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX8-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX8-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX900-LABEL: xnor_i64_s_v_one_use:
; GFX900: ; %bb.0: ; %entry
; GFX900-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX900-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX900-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX900-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX900-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX900-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX900-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX900-NEXT: ; return to shader part epilog
;
; GFX906-LABEL: xnor_i64_s_v_one_use:
; GFX906: ; %bb.0: ; %entry
; GFX906-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX906-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX906-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX906-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX906-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX906-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX906-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX906-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: xnor_i64_s_v_one_use:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX10-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX10-NEXT: v_xor_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX10-NEXT: v_xor3_b32 v0, s0, v0, -1
+; GFX10-NEXT: v_xor3_b32 v1, s1, v1, -1
; GFX10-NEXT: ; return to shader part epilog
entry:
%b = shl i64 %b64, 29
@@ -375,41 +353,44 @@ define amdgpu_ps <2 x float> @xnor_i64_v_s_one_use(i64 inreg %a, i64 %b64) {
; GFX7-LABEL: xnor_i64_v_s_one_use:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_lshl_b64 v[0:1], v[0:1], 29
-; GFX7-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX7-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX7-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX7-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX7-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX7-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: xnor_i64_v_s_one_use:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX8-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX8-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX8-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX8-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX900-LABEL: xnor_i64_v_s_one_use:
; GFX900: ; %bb.0:
; GFX900-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX900-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX900-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX900-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX900-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX900-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX900-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX900-NEXT: ; return to shader part epilog
;
; GFX906-LABEL: xnor_i64_v_s_one_use:
; GFX906: ; %bb.0:
; GFX906-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX906-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX906-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX906-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX906-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX906-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX906-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX906-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: xnor_i64_v_s_one_use:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 29, v[0:1]
-; GFX10-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX10-NEXT: v_xor_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_xor_b32_e32 v1, s1, v1
+; GFX10-NEXT: v_xor3_b32 v0, v0, s0, -1
+; GFX10-NEXT: v_xor3_b32 v1, v1, s1, -1
; GFX10-NEXT: ; return to shader part epilog
%b = shl i64 %b64, 29
%xor = xor i64 %b, %a
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
index e544df0c6e652..4755da1392684 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
define amdgpu_ps i16 @s_xor_i16(i16 inreg %num, i16 inreg %den) {
; GCN-LABEL: s_xor_i16:
@@ -532,8 +532,7 @@ define amdgpu_ps void @s_xor_u64_zext_with_vregs(ptr addrspace(1) %out, ptr addr
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: buffer_load_dword v2, v[2:3], s[0:3], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v3, 0
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -689,8 +688,7 @@ define amdgpu_ps void @s_xor_u64_sext_with_vregs(ptr addrspace(1) %out, ptr addr
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: buffer_load_dword v2, v[2:3], s[0:3], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_ashrrev_i32_e32 v3, 31, v2
@@ -755,11 +753,11 @@ define amdgpu_kernel void @s_xor_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_ashr_i32 s5, s4, 31
-; GFX7-NEXT: s_xor_b32 s4, s4, 0x50
+; GFX7-NEXT: s_xor_b64 s[4:5], s[4:5], 0x50
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -769,15 +767,15 @@ define amdgpu_kernel void @s_xor_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_load_dword s2, s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_ashr_i32 s0, s2, 31
-; GFX8-NEXT: s_xor_b32 s1, s2, 0x50
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, s0
-; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT: s_ashr_i32 s3, s2, 31
+; GFX8-NEXT: s_xor_b64 s[2:3], s[2:3], 0x50
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: s_xor_u64_sext_with_sregs:
@@ -788,7 +786,7 @@ define amdgpu_kernel void @s_xor_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX9-NEXT: s_load_dword s2, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_ashr_i32 s3, s2, 31
-; GFX9-NEXT: s_xor_b32 s2, s2, 0x50
+; GFX9-NEXT: s_xor_b64 s[2:3], s[2:3], 0x50
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -801,37 +799,37 @@ define amdgpu_kernel void @s_xor_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s2, s[2:3], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_xor_b32 s3, s2, 0x50
-; GFX10-NEXT: s_ashr_i32 s2, s2, 31
-; GFX10-NEXT: v_mov_b32_e32 v0, s3
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_ashr_i32 s3, s2, 31
+; GFX10-NEXT: s_xor_b64 s[2:3], s[2:3], 0x50
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_xor_u64_sext_with_sregs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_xor_b32 s3, s2, 0x50
-; GFX11-NEXT: s_ashr_i32 s2, s2, 31
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s3
+; GFX11-NEXT: s_ashr_i32 s3, s2, 31
+; GFX11-NEXT: s_xor_b64 s[2:3], s[2:3], 0x50
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_xor_u64_sext_with_sregs:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_xor_b32 s3, s2, 0x50
-; GFX12-NEXT: s_ashr_i32 s2, s2, 31
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, s3
+; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_xor_b64 s[2:3], s[2:3], 0x50
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/add-max.ll b/llvm/test/CodeGen/AMDGPU/add-max.ll
index 9a49e9d8c725a..2af97c3d9c301 100644
--- a/llvm/test/CodeGen/AMDGPU/add-max.ll
+++ b/llvm/test/CodeGen/AMDGPU/add-max.ll
@@ -171,13 +171,30 @@ define amdgpu_ps float @add_max_v2u16_ssv(<2 x i16> inreg %a, <2 x i16> inreg %b
}
define amdgpu_ps float @add_max_v2u16_sss(<2 x i16> inreg %a, <2 x i16> inreg %b, <2 x i16> inreg %c) {
-; GCN-LABEL: add_max_v2u16_sss:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: v_pk_add_u16 v0, s0, s1 clamp
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT: v_pk_max_u16 v0, v0, s2
-; GCN-NEXT: ; return to shader part epilog
+; SDAG-LABEL: add_max_v2u16_sss:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: v_pk_add_u16 v0, s0, s1 clamp
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_pk_max_u16 v0, v0, s2
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-LABEL: add_max_v2u16_sss:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: v_pk_add_u16 v0, s0, s1 clamp
+; GISEL-NEXT: s_and_b32 s1, s2, 0xffff
+; GISEL-NEXT: s_lshr_b32 s2, s2, 16
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: s_and_b32 s3, s0, 0xffff
+; GISEL-NEXT: s_lshr_b32 s0, s0, 16
+; GISEL-NEXT: s_max_u32 s1, s3, s1
+; GISEL-NEXT: s_max_u32 s0, s0, s2
+; GISEL-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: ; return to shader part epilog
%add = call <2 x i16> @llvm.uadd.sat.i32(<2 x i16> %a, <2 x i16> %b)
%max = call <2 x i16> @llvm.umax.v216(<2 x i16> %add, <2 x i16> %c)
%ret = bitcast <2 x i16> %max to float
diff --git a/llvm/test/CodeGen/AMDGPU/bfi_int.ll b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
index da66ae414dcd2..c6bdb5ad928b1 100644
--- a/llvm/test/CodeGen/AMDGPU/bfi_int.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
@@ -644,15 +644,25 @@ define i64 @v_bitselect_i64_pat_0(i64 %a, i64 %b, i64 %mask) {
; GFX8-GISEL-LABEL: v_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, v3, v5
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, v0, v2, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: v_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, v0, v2, v4
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, v1, v3, v5
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v6, -1, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v7, -1, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, v6, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, v7, v5
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, v0, v2, v4
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, v1, v3, v5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -686,16 +696,24 @@ define amdgpu_ps <2 x float> @v_s_s_bitselect_i64_pat_0(i64 %a, i64 inreg %b, i6
;
; GFX8-GISEL-LABEL: v_s_s_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, s1, v2
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, v0, s0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, s1, v1
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, s3, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: v_s_s_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, v0, s0, s2
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, v1, s1, s3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, -1, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v3, -1, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, s2, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, s3, v3
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, v0, s0, v2
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, v1, s1, v3
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -730,16 +748,22 @@ define amdgpu_ps <2 x float> @s_v_s_bitselect_i64_pat_0(i64 inreg %a, i64 %b, i6
;
; GFX8-GISEL-LABEL: s_v_s_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, s1, v1, v2
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, s1, v1
+; GFX8-GISEL-NEXT: s_andn2_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_v_s_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, s0, v0, s2
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, s1, v1, s3
+; GFX10-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, s0, v0, v2
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, s1, v1, v3
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -774,16 +798,28 @@ define amdgpu_ps <2 x float> @s_s_v_bitselect_i64_pat_0(i64 inreg %a, i64 inreg
;
; GFX8-GISEL-LABEL: s_s_v_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, s0, v2, v0
+; GFX8-GISEL-NEXT: s_and_b64 s[2:3], s[0:1], s[2:3]
+; GFX8-GISEL-NEXT: s_not_b64 s[0:1], s[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_s_v_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, s0, s2, v0
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, s1, s3, v1
+; GFX10-GISEL-NEXT: s_not_b64 s[4:5], s[0:1]
+; GFX10-GISEL-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s0
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, v3, v1, v5
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, v2, v0, v4
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -814,14 +850,24 @@ define amdgpu_ps <2 x float> @v_v_s_bitselect_i64_pat_0(i64 %a, i64 %b, i64 inre
;
; GFX8-GISEL-LABEL: v_v_s_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, v3, s1
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, v0, v2, s0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, s1, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: v_v_s_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, v0, v2, s0
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, v1, v3, s1
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v4, -1, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v5, -1, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, s0, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, s1, v5
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, v0, v2, v4
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, v1, v3, v5
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -852,14 +898,24 @@ define amdgpu_ps <2 x float> @v_s_v_bitselect_i64_pat_0(i64 %a, i64 inreg %b, i6
;
; GFX8-GISEL-LABEL: v_s_v_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, s1, v3
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, v0, s0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v4, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v5, s1, v1
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v5, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: v_s_v_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, v0, s0, v2
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, v1, s1, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v4, -1, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v5, -1, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v4, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v5, v3
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, v0, s0, v2
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, v1, s1, v3
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -890,14 +946,26 @@ define amdgpu_ps <2 x float> @s_v_v_bitselect_i64_pat_0(i64 inreg %a, i64 %b, i6
;
; GFX8-GISEL-LABEL: s_v_v_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, s1, v1, v3
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v4, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v5, s1, v1
+; GFX8-GISEL-NEXT: s_not_b64 s[0:1], s[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v5, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_v_v_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v2
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, s1, v1, v3
+; GFX10-GISEL-NEXT: s_not_b64 s[2:3], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v5, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v4, v2
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, s1, v1, v3
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, s0, v0, v2
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -973,10 +1041,10 @@ define amdgpu_ps <2 x float> @v_s_s_bitselect_i64_pat_1(i64 %a, i64 inreg %b, i6
;
; GFX8-GISEL-LABEL: v_s_s_bitselect_i64_pat_1:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, s1, v1, v2
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v2
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT: v_bfi_b32 v0, v2, v0, s2
+; GFX8-GISEL-NEXT: v_bfi_b32 v1, v3, v1, s3
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: v_s_s_bitselect_i64_pat_1:
@@ -1016,10 +1084,10 @@ define amdgpu_ps <2 x float> @s_s_v_bitselect_i64_pat_1(i64 inreg %a, i64 inreg
;
; GFX8-GISEL-LABEL: s_s_v_bitselect_i64_pat_1:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, s3, v2, v1
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, s2, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX8-GISEL-NEXT: v_bfi_b32 v0, v2, s0, v0
+; GFX8-GISEL-NEXT: v_bfi_b32 v1, v3, s1, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_s_v_bitselect_i64_pat_1:
@@ -1059,16 +1127,24 @@ define amdgpu_ps <2 x float> @s_v_s_bitselect_i64_pat_1(i64 inreg %a, i64 %b, i6
;
; GFX8-GISEL-LABEL: s_v_s_bitselect_i64_pat_1:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, s1, v2
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, v0, s0, v2
+; GFX8-GISEL-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v3, v1
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, s3, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_v_s_bitselect_i64_pat_1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, v0, s0, s2
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, v1, s1, s3
+; GFX10-GISEL-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v3, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, s3, v1
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, s2, v0
; GFX10-GISEL-NEXT: ; return to shader part epilog
%xor.0 = xor i64 %a, %mask
%and = and i64 %xor.0, %b
@@ -1202,12 +1278,12 @@ define amdgpu_ps <2 x float> @v_s_s_bfi_sha256_ma_i64(i64 %x, i64 inreg %y, i64
;
; GFX8-GISEL-LABEL: v_s_s_bfi_sha256_ma_i64:
; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, s1, v1
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, s3, v2
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, s0, v0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX8-GISEL-NEXT: v_bfi_b32 v0, v0, s2, v2
+; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, s3, v3
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: v_s_s_bfi_sha256_ma_i64:
@@ -1253,18 +1329,28 @@ define amdgpu_ps <2 x float> @s_v_s_bfi_sha256_ma_i64(i64 inreg %x, i64 %y, i64
;
; GFX8-GISEL-LABEL: s_v_s_bfi_sha256_ma_i64:
; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, s1, v1
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v2, s3, v1
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, s0, v0
-; GFX8-GISEL-NEXT: v_bfi_b32 v0, v2, s2, v0
+; GFX8-GISEL-NEXT: s_and_b64 s[4:5], s[0:1], s[2:3]
+; GFX8-GISEL-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v1, v3
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_v_s_bfi_sha256_ma_i64:
; GFX10-GISEL: ; %bb.0: ; %entry
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, s0, v0
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v3, s1, v1
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, v2, s2, v0
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, v3, s3, v1
+; GFX10-GISEL-NEXT: s_or_b64 s[4:5], s[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s0
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, v1, v3, v5
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, v0, v2, v4
; GFX10-GISEL-NEXT: ; return to shader part epilog
entry:
%and0 = and i64 %x, %z
@@ -1306,12 +1392,12 @@ define amdgpu_ps <2 x float> @s_s_v_bfi_sha256_ma_i64(i64 inreg %x, i64 inreg %y
;
; GFX8-GISEL-LABEL: s_s_v_bfi_sha256_ma_i64:
; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, s1, v2
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v2, v1, s3
; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, s0, v2
; GFX8-GISEL-NEXT: v_bfi_b32 v0, v2, v0, s2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, s1, v3
+; GFX8-GISEL-NEXT: v_bfi_b32 v1, v2, v1, s3
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_s_v_bfi_sha256_ma_i64:
@@ -1357,10 +1443,10 @@ define amdgpu_ps <2 x float> @v_s_v_bfi_sha256_ma_i64(i64 %x, i64 inreg %y, i64
;
; GFX8-GISEL-LABEL: v_s_v_bfi_sha256_ma_i64:
; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, s0, v0
-; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, v3, s1
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, s1, v1
; GFX8-GISEL-NEXT: v_bfi_b32 v0, v0, v2, s0
+; GFX8-GISEL-NEXT: v_bfi_b32 v1, v1, v3, s1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: v_s_v_bfi_sha256_ma_i64:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index 8132fed599f70..f8cbd5747eefa 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -3458,7 +3458,7 @@ define <2 x i6> @load_v2i6(ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GISEL-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GISEL-NEXT: s_and_b32 s4, 0xffff, s4
; GISEL-NEXT: s_lshr_b32 s4, s4, 6
; GISEL-NEXT: v_mov_b32_e32 v1, s4
; GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
index fef98ce3bb75e..f8781afe9effc 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
@@ -63,20 +63,20 @@ define amdgpu_kernel void @buffer_structs(%fat_buffer_struct %arg, ptr addrspace
; GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
-; GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-NEXT: v_mov_b32_e32 v5, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: s_ashr_i32 s7, s6, 31
; GISEL-NEXT: s_lshl_b64 s[4:5], s[6:7], 5
; GISEL-NEXT: s_add_u32 s4, s8, s4
-; GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GISEL-NEXT: v_mov_b32_e32 v4, s6
; GISEL-NEXT: s_addc_u32 s5, s9, s5
-; GISEL-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen
-; GISEL-NEXT: global_store_dword v4, v0, s[4:5] offset:16
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: v_mov_b32_e32 v1, s1
; GISEL-NEXT: v_mov_b32_e32 v2, s2
; GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
+; GISEL-NEXT: buffer_store_dword v4, v4, s[0:3], 0 offen
+; GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[4:5]
+; GISEL-NEXT: global_store_dword v5, v4, s[4:5] offset:16
; GISEL-NEXT: s_endpgm
;
; SDAG-LABEL: buffer_structs:
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index 02804a27f3fd7..594f036b391ed 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -862,10 +862,10 @@ define amdgpu_kernel void @v_ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspa
; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
-; GFX10-GISEL-NEXT: v_min3_u32 v0, v0, v1, 64
+; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v3, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
+; GFX10-GISEL-NEXT: v_min3_u32 v0, v3, v0, 64
; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
;
@@ -991,7 +991,7 @@ define amdgpu_kernel void @v_ctlz_i64_trunc(ptr addrspace(1) noalias %out, ptr a
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v1, v1, 32 clamp
-; GFX10-GISEL-NEXT: v_min3_u32 v1, v1, v2, 64
+; GFX10-GISEL-NEXT: v_min3_u32 v1, v2, v1, 64
; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
;
@@ -1738,18 +1738,18 @@ define amdgpu_kernel void @v_ctlz_i64_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
+; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
-; GFX10-GISEL-NEXT: v_min3_u32 v0, v0, v1, 64
-; GFX10-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 64, v0
-; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, -1, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_min3_u32 v0, v2, v0, 64
+; GFX10-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 64, v[0:1]
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v3, -1, 0, vcc_lo
+; GFX10-GISEL-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i64_sel_ne_bitwidth:
@@ -2048,11 +2048,20 @@ define amdgpu_kernel void @v_ctlz_i8_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-LABEL: v_ctlz_i8_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
+; GFX10-GISEL-NEXT: v_add_nc_u16 v1, 0xffe8, v1
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0xffff, vcc_lo
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
;
@@ -2158,7 +2167,14 @@ define amdgpu_kernel void @v_ctlz_i8_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-GISEL-NEXT: s_flbit_i32_b32 s3, s2
+; GFX10-GISEL-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX10-GISEL-NEXT: s_min_u32 s3, s3, 32
+; GFX10-GISEL-NEXT: s_add_i32 s3, s3, 0xfff0
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s2, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s2, 0xffff, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX10-GISEL-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
;
@@ -2259,11 +2275,21 @@ define amdgpu_kernel void @v_ctlz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-LABEL: v_ctlz_i7_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0
+; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
+; GFX10-GISEL-NEXT: v_add_nc_u16 v1, 0xffe7, v1
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0x7f, vcc_lo
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0
; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
index ac50015a10aa2..af445d6a054f1 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
@@ -771,7 +771,7 @@ define amdgpu_kernel void @v_ctlz_zero_poison_i16_with_select(ptr addrspace(1) n
; GFX9-GISEL-NEXT: s_or_b32 s2, s3, s2
; GFX9-GISEL-NEXT: s_lshl_b32 s3, s2, 16
; GFX9-GISEL-NEXT: s_flbit_i32_b32 s3, s3
-; GFX9-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX9-GISEL-NEXT: s_and_b32 s2, 0xffff, s2
; GFX9-GISEL-NEXT: s_cselect_b32 s2, s3, 32
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
@@ -1179,9 +1179,14 @@ define amdgpu_kernel void @v_ctlz_zero_poison_i8(ptr addrspace(1) noalias %out,
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_i8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
+; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
@@ -1380,16 +1385,16 @@ define amdgpu_kernel void @v_ctlz_zero_poison_i64(ptr addrspace(1) noalias %out,
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_i64:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v3, s[2:3]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX9-GISEL-NEXT: v_add_u32_e32 v0, 32, v0
-; GFX9-GISEL-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: v_min_u32_e32 v1, v1, v0
+; GFX9-GISEL-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
@@ -1476,9 +1481,9 @@ define amdgpu_kernel void @v_ctlz_zero_poison_i64_trunc(ptr addrspace(1) noalias
; GFX9-GISEL-NEXT: global_load_dwordx2 v[1:2], v1, s[2:3]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
-; GFX9-GISEL-NEXT: v_min_u32_e32 v1, v1, v2
+; GFX9-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX9-GISEL-NEXT: v_min_u32_e32 v1, v2, v1
; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1713,11 +1718,20 @@ define amdgpu_kernel void @v_ctlz_zero_poison_i8_sel_eq_neg1(ptr addrspace(1) no
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_i8_sel_eq_neg1:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
+; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index fac1e1f6ec635..ae534a1401db0 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -2331,32 +2331,30 @@ define amdgpu_kernel void @ctpop_i16_in_br(ptr addrspace(1) %out, ptr addrspace(
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: s_lshr_b32 s4, s6, 16
+; VI-GISEL-NEXT: s_mov_b32 s4, 1
; VI-GISEL-NEXT: s_cbranch_scc0 .LBB14_2
; VI-GISEL-NEXT: ; %bb.1: ; %else
-; VI-GISEL-NEXT: s_mov_b32 s11, 0xf000
; VI-GISEL-NEXT: s_mov_b32 s10, -1
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
+; VI-GISEL-NEXT: s_mov_b32 s11, 0xf000
+; VI-GISEL-NEXT: s_mov_b64 s[8:9], s[2:3]
; VI-GISEL-NEXT: buffer_load_ushort v0, off, s[8:11], 0 offset:2
-; VI-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; VI-GISEL-NEXT: s_mov_b32 s4, 0
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_readfirstlane_b32 s2, v0
; VI-GISEL-NEXT: s_branch .LBB14_3
; VI-GISEL-NEXT: .LBB14_2:
-; VI-GISEL-NEXT: s_mov_b64 s[2:3], -1
-; VI-GISEL-NEXT: ; implicit-def: $vgpr0
+; VI-GISEL-NEXT: ; implicit-def: $sgpr2
; VI-GISEL-NEXT: .LBB14_3: ; %Flow
-; VI-GISEL-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; VI-GISEL-NEXT: s_cmp_lg_u32 s2, 1
+; VI-GISEL-NEXT: s_xor_b32 s3, s4, 1
+; VI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
; VI-GISEL-NEXT: s_cbranch_scc1 .LBB14_5
; VI-GISEL-NEXT: ; %bb.4: ; %if
; VI-GISEL-NEXT: s_and_b32 s2, s6, 0xffff
; VI-GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: .LBB14_5: ; %endif
-; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: s_mov_b32 s2, -1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop64.ll b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
index b7ede7d793c11..60b95218105eb 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
@@ -98,15 +98,15 @@ define amdgpu_kernel void @v_ctpop_i64(ptr addrspace(1) noalias %out, ptr addrsp
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[0:1], v0, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v1, v0
-; GFX12-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-NEXT: v_bcnt_u32_b32 v1, v1, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
@@ -172,8 +172,8 @@ define amdgpu_kernel void @v_ctpop_i64_user(ptr addrspace(1) noalias %out, ptr a
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[0:1], v0, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v1, v0
+; GFX12-NEXT: v_bcnt_u32_b32 v1, v1, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, v1
; GFX12-NEXT: v_mov_b32_e32 v1, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_or_b32_e32 v0, s4, v0
@@ -340,18 +340,18 @@ define amdgpu_kernel void @v_ctpop_v2i64(ptr addrspace(1) noalias %out, ptr addr
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b128 v[0:3], v0, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX12-NEXT: v_bcnt_u32_b32 v2, v2, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v1, v1, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v3, v3, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v1, v0
-; GFX12-NEXT: v_bcnt_u32_b32 v1, v3, v2
-; GFX12-NEXT: global_store_b64 v4, v[0:1], s[0:1]
+; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, v1
+; GFX12-NEXT: v_bcnt_u32_b32 v1, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <2 x i64>, ptr addrspace(1) %in, i32 %tid
@@ -422,25 +422,25 @@ define amdgpu_kernel void @v_ctpop_v4i64(ptr addrspace(1) noalias %out, ptr addr
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: v_mov_b32_e32 v8, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e32 v4, 5, v0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_load_b128 v[0:3], v4, s[2:3]
; GFX12-NEXT: global_load_b128 v[4:7], v4, s[2:3] offset:16
; GFX12-NEXT: s_wait_loadcnt 0x1
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX12-NEXT: v_bcnt_u32_b32 v2, v2, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v1, v1, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v3, v3, 0
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_bcnt_u32_b32 v4, v4, 0
-; GFX12-NEXT: v_bcnt_u32_b32 v6, v6, 0
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v1, v0
-; GFX12-NEXT: v_bcnt_u32_b32 v1, v3, v2
+; GFX12-NEXT: v_bcnt_u32_b32 v5, v5, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v7, v7, 0
+; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, v1
+; GFX12-NEXT: v_bcnt_u32_b32 v1, v2, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_bcnt_u32_b32 v2, v5, v4
-; GFX12-NEXT: v_bcnt_u32_b32 v3, v7, v6
-; GFX12-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-NEXT: v_bcnt_u32_b32 v2, v4, v5
+; GFX12-NEXT: v_bcnt_u32_b32 v3, v6, v7
+; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <4 x i64>, ptr addrspace(1) %in, i32 %tid
@@ -728,13 +728,13 @@ define amdgpu_kernel void @v_ctpop_i128(ptr addrspace(1) noalias %out, ptr addrs
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b128 v[0:3], v0, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_bcnt_u32_b32 v1, v1, 0
; GFX12-NEXT: v_bcnt_u32_b32 v2, v2, 0
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_bcnt_u32_b32 v2, v3, v2
-; GFX12-NEXT: v_bcnt_u32_b32 v0, v1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v2
+; GFX12-NEXT: v_bcnt_u32_b32 v3, v3, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_bcnt_u32_b32 v0, v0, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: v_add3_u32 v0, v2, v3, v0
; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index e4adcd9409238..6cab7f3edde31 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -750,9 +750,9 @@ define amdgpu_kernel void @v_cttz_i64(ptr addrspace(1) noalias %out, ptr addrspa
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
-; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v1, v1, 32 clamp
-; GFX10-GISEL-NEXT: v_min3_u32 v0, v0, v1, 64
+; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v3, v1, 32 clamp
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_min3_u32 v0, v0, v3, 64
; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1347,11 +1347,19 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX10-GISEL-LABEL: v_cttz_i8_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, 0x100, v0
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0xffff, vcc_lo
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1448,7 +1456,7 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_readfirstlane_b32 s2, v1
; GFX10-GISEL-NEXT: s_or_b32 s3, s2, 0x10000
-; GFX10-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX10-GISEL-NEXT: s_and_b32 s2, 0xffff, s2
; GFX10-GISEL-NEXT: s_ff1_i32_b32 s3, s3
; GFX10-GISEL-NEXT: s_cmp_eq_u32 s2, 0
; GFX10-GISEL-NEXT: s_cselect_b32 s2, 0xffff, s3
@@ -1542,12 +1550,20 @@ define amdgpu_kernel void @v_cttz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-LABEL: v_cttz_i7_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, 0x80, v0
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0
+; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0x7f, vcc_lo
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll
index 9903e9a8b11b1..1695100ae231a 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll
@@ -736,7 +736,7 @@ define amdgpu_kernel void @v_cttz_zero_poison_i16_with_select(ptr addrspace(1) n
; GFX9-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX9-GISEL-NEXT: s_or_b32 s2, s3, s2
; GFX9-GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX9-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX9-GISEL-NEXT: s_and_b32 s2, 0xffff, s2
; GFX9-GISEL-NEXT: s_cselect_b32 s2, s3, 32
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
@@ -1642,7 +1642,7 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX9-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX9-GISEL-NEXT: s_or_b32 s2, s3, s2
; GFX9-GISEL-NEXT: s_or_b32 s3, s2, 0x10000
-; GFX9-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX9-GISEL-NEXT: s_and_b32 s2, 0xffff, s2
; GFX9-GISEL-NEXT: s_ff1_i32_b32 s3, s3
; GFX9-GISEL-NEXT: s_cmp_eq_u32 s2, 0
; GFX9-GISEL-NEXT: s_cselect_b32 s2, 0xffff, s3
diff --git a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
index d716696754f4d..4ddf17c8aaa9c 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
@@ -51,7 +51,7 @@ define amdgpu_kernel void @ds2align1(ptr addrspace(3) %in, ptr addrspace(3) %out
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s1, v0
; ALIGNED-GISEL-NEXT: s_lshl_b32 s1, s1, 8
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s1, s0
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s1, s0, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s0
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s1, 8
; ALIGNED-GISEL-NEXT: ds_write_b8 v2, v0
@@ -134,7 +134,7 @@ define amdgpu_kernel void @ds4align1(ptr addrspace(3) %in, ptr addrspace(3) %out
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
; ALIGNED-GISEL-NEXT: s_or_b32 s2, s4, s3
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s3, s0, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s3, 0xffff, s0
; ALIGNED-GISEL-NEXT: s_lshr_b32 s3, s3, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -299,7 +299,7 @@ define amdgpu_kernel void @ds8align1(ptr addrspace(3) %in, ptr addrspace(3) %out
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24
; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s4, s0, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s4, 0xffff, s0
; ALIGNED-GISEL-NEXT: s_lshr_b32 s4, s4, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -312,7 +312,7 @@ define amdgpu_kernel void @ds8align1(ptr addrspace(3) %in, ptr addrspace(3) %out
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:2
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s1, s2, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s2
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:3
; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2
@@ -535,7 +535,7 @@ define amdgpu_kernel void @ds12align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 24
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16
; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s5, s0, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s5, 0xffff, s0
; ALIGNED-GISEL-NEXT: s_lshr_b32 s5, s5, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -548,7 +548,7 @@ define amdgpu_kernel void @ds12align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s4
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:2
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s1, s2, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s2
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:3
; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2
@@ -560,7 +560,7 @@ define amdgpu_kernel void @ds12align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:6
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s1, s3, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s3
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:7
; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3
@@ -909,7 +909,7 @@ define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: s_lshl_b32 s6, s6, 24
; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 16
; ALIGNED-GISEL-NEXT: s_or_b32 s5, s6, s5
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s6, s0, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s6, 0xffff, s0
; ALIGNED-GISEL-NEXT: s_lshr_b32 s6, s6, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -922,7 +922,7 @@ define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s5
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:2
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s1, s2, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s2
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:3
; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2
@@ -934,7 +934,7 @@ define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:6
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s1, s3, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s3
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:7
; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3
@@ -946,7 +946,7 @@ define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:10
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; ALIGNED-GISEL-NEXT: s_bfe_u32 s1, s4, 0x100000
+; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s4
; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:11
; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s4
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 34641c39ee128..38dc8a665aeae 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -25,19 +25,20 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_uniform(i32 %n) #0 {
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_uniform:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_load_dword s5, s[8:9], 0x0
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s17
-; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x400
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s32
+; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_lshl2_add_u32 s4, s4, 15
-; GFX9-GISEL-NEXT: s_and_b32 s4, s4, -16
+; GFX9-GISEL-NEXT: s_lshl2_add_u32 s5, s5, 15
+; GFX9-GISEL-NEXT: s_and_b32 s5, s5, -16
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x7b
-; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-GISEL-NEXT: s_lshl_b32 s5, s5, 6
; GFX9-GISEL-NEXT: s_mov_b32 s33, 0
-; GFX9-GISEL-NEXT: s_add_i32 s32, s5, s4
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX9-GISEL-NEXT: s_add_u32 s32, s4, s5
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_endpgm
;
@@ -104,16 +105,16 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_uniform_over_aligned(i
; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x2000
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s17
; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-GISEL-NEXT: s_add_i32 s5, s32, 0x1fff
+; GFX9-GISEL-NEXT: s_add_u32 s5, s32, 0x1fff
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xffffe000
; GFX9-GISEL-NEXT: s_and_b32 s4, s4, -16
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 10
-; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 6
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 6
; GFX9-GISEL-NEXT: s_mov_b32 s33, 0
-; GFX9-GISEL-NEXT: s_add_i32 s32, s5, s4
+; GFX9-GISEL-NEXT: s_add_u32 s32, s5, s4
; GFX9-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_endpgm
@@ -177,19 +178,20 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_uniform_under_aligned(
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_uniform_under_aligned:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_load_dword s5, s[8:9], 0x0
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s17
-; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x400
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s32
+; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_lshl2_add_u32 s4, s4, 15
-; GFX9-GISEL-NEXT: s_and_b32 s4, s4, -16
+; GFX9-GISEL-NEXT: s_lshl2_add_u32 s5, s5, 15
+; GFX9-GISEL-NEXT: s_and_b32 s5, s5, -16
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 22
-; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-GISEL-NEXT: s_lshl_b32 s5, s5, 6
; GFX9-GISEL-NEXT: s_mov_b32 s33, 0
-; GFX9-GISEL-NEXT: s_add_i32 s32, s5, s4
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX9-GISEL-NEXT: s_add_u32 s32, s4, s5
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_endpgm
;
@@ -265,12 +267,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent() #0 {
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent:
; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x400
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s17
; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-GISEL-NEXT: s_mov_b32 s33, 0
-; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x400
; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
; GFX9-GISEL-NEXT: s_nop 1
@@ -285,14 +288,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent() #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x7b
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_endpgm
;
@@ -404,7 +406,7 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_over_aligned
; GFX9-GISEL-NEXT: s_mov_b32 s33, 0
; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x2000
; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
; GFX9-GISEL-NEXT: s_nop 1
@@ -421,13 +423,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_over_aligned
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_add_i32 s4, s32, 0x1fff
-; GFX9-GISEL-NEXT: s_and_b32 s4, s4, 0xffffe000
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v2, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x1bc
-; GFX9-GISEL-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GFX9-GISEL-NEXT: s_add_u32 s5, s32, 0x1fff
+; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xffffe000
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s6, 6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x1bc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-GISEL-NEXT: s_add_u32 s32, s5, s4
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_endpgm
;
@@ -535,11 +537,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_under_aligne
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent_under_aligned:
; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x400
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s17
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 4, 15
; GFX9-GISEL-NEXT: s_mov_b32 s33, 0
-; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x400
; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
; GFX9-GISEL-NEXT: s_nop 1
@@ -554,14 +558,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_under_aligne
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x29a
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_endpgm
;
@@ -703,13 +706,14 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX9-GISEL-NEXT: s_cbranch_scc1 .LBB6_2
; GFX9-GISEL-NEXT: ; %bb.1: ; %bb.0
; GFX9-GISEL-NEXT: s_lshl2_add_u32 s5, s5, 15
-; GFX9-GISEL-NEXT: s_add_i32 s6, s32, 0xfff
; GFX9-GISEL-NEXT: s_and_b32 s5, s5, -16
-; GFX9-GISEL-NEXT: s_and_b32 s8, s6, 0xfffff000
+; GFX9-GISEL-NEXT: s_add_u32 s6, s32, 0xfff
; GFX9-GISEL-NEXT: s_lshl_b32 s5, s5, 6
+; GFX9-GISEL-NEXT: s_and_b32 s8, s6, 0xfffff000
+; GFX9-GISEL-NEXT: s_add_u32 s32, s8, s5
; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
-; GFX9-GISEL-NEXT: s_add_i32 s32, s8, s5
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
+; GFX9-GISEL-NEXT: s_mov_b32 s5, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_nop 1
@@ -724,30 +728,30 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s5, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s9, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s5, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 3
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s9, 6
; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 4
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-GISEL-NEXT: s_add_u32 s32, s5, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: .LBB6_2: ; %bb.1
; GFX9-GISEL-NEXT: s_lshl2_add_u32 s4, s4, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s5, s32
; GFX9-GISEL-NEXT: s_and_b32 s4, s4, -16
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 1
; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 6
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s32
; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s33
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 2
-; GFX9-GISEL-NEXT: s_add_i32 s32, s5, s4
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-GISEL-NEXT: s_add_u32 s32, s5, s4
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_endpgm
;
@@ -946,13 +950,16 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX9-GISEL-NEXT: s_movk_i32 s32, 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 1
; GFX9-GISEL-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-GISEL-NEXT: ; %bb.1: ; %bb.1
; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[6:7]
-; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT: s_nop 0
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
@@ -964,36 +971,30 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s4, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s9, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s4, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s9, 6
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 1
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-GISEL-NEXT: s_add_u32 s32, s8, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b64 s[6:7], 0
-; GFX9-GISEL-NEXT: s_branch .LBB7_3
-; GFX9-GISEL-NEXT: .LBB7_2:
-; GFX9-GISEL-NEXT: s_mov_b64 s[6:7], -1
-; GFX9-GISEL-NEXT: .LBB7_3: ; %Flow
-; GFX9-GISEL-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX9-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX9-GISEL-NEXT: s_cmp_lg_u32 s4, 1
-; GFX9-GISEL-NEXT: s_cbranch_scc1 .LBB7_5
-; GFX9-GISEL-NEXT: ; %bb.4: ; %bb.0
-; GFX9-GISEL-NEXT: s_add_i32 s4, s32, 0xfff
-; GFX9-GISEL-NEXT: s_lshl2_add_u32 s5, s5, 15
-; GFX9-GISEL-NEXT: s_and_b32 s4, s4, 0xfffff000
-; GFX9-GISEL-NEXT: s_and_b32 s5, s5, -16
-; GFX9-GISEL-NEXT: s_lshl_b32 s5, s5, 6
+; GFX9-GISEL-NEXT: .LBB7_2: ; %Flow
+; GFX9-GISEL-NEXT: s_xor_b32 s4, s4, 1
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-GISEL-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX9-GISEL-NEXT: ; %bb.3: ; %bb.0
+; GFX9-GISEL-NEXT: s_lshl2_add_u32 s4, s5, 15
+; GFX9-GISEL-NEXT: s_add_u32 s5, s32, 0xfff
+; GFX9-GISEL-NEXT: s_and_b32 s4, s4, -16
+; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xfffff000
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 6
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-GISEL-NEXT: s_add_i32 s32, s4, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-GISEL-NEXT: s_add_u32 s32, s5, s4
; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: .LBB7_5: ; %bb.2
+; GFX9-GISEL-NEXT: .LBB7_4: ; %bb.2
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: test_dynamic_stackalloc_kernel_control_flow:
@@ -1166,13 +1167,14 @@ define void @test_dynamic_stackalloc_device_uniform(i32 %n) #0 {
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_uniform:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s7, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s33
; GFX9-GISEL-NEXT: s_mov_b32 s33, s32
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x400
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
@@ -1188,20 +1190,19 @@ define void @test_dynamic_stackalloc_device_uniform(i32 %n) #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x7b
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s33
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s7
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s8
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1376,13 +1377,13 @@ define void @test_dynamic_stackalloc_device_uniform_over_aligned(i32 %n) #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_add_i32 s4, s32, 0x1fff
-; GFX9-GISEL-NEXT: s_and_b32 s4, s4, 0xffffe000
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v2, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 10
-; GFX9-GISEL-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GFX9-GISEL-NEXT: s_add_u32 s5, s32, 0x1fff
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s6, 6
+; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xffffe000
+; GFX9-GISEL-NEXT: s_add_u32 s32, s5, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 10
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s34
; GFX9-GISEL-NEXT: s_mov_b32 s34, s8
@@ -1545,13 +1546,14 @@ define void @test_dynamic_stackalloc_device_uniform_under_aligned(i32 %n) #0 {
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_uniform_under_aligned:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s7, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s33
; GFX9-GISEL-NEXT: s_mov_b32 s33, s32
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x400
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
@@ -1567,20 +1569,19 @@ define void @test_dynamic_stackalloc_device_uniform_under_aligned(i32 %n) #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 22
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s33
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s7
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s8
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1724,15 +1725,16 @@ define void @test_dynamic_stackalloc_device_divergent() #0 {
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_divergent:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s7, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s33
; GFX9-GISEL-NEXT: s_mov_b32 s33, s32
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v31
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v1, 2, 15
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x400
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x1ff0, v1
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v1, 2, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, -16, v1
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_nop 1
@@ -1747,20 +1749,19 @@ define void @test_dynamic_stackalloc_device_divergent() #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v0, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v0, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, s6, 6, v1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x7b
-; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s33
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s7
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s8
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1913,20 +1914,18 @@ define void @test_dynamic_stackalloc_device_divergent_over_aligned() #0 {
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_divergent_over_aligned:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s7, s33
; GFX9-GISEL-NEXT: s_add_i32 s33, s32, 0x1fc0
; GFX9-GISEL-NEXT: s_and_b32 s33, s33, 0xffffe000
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s34
-; GFX9-GISEL-NEXT: s_mov_b32 s34, s32
-; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x4000
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v31
-; GFX9-GISEL-NEXT: s_add_i32 s4, s32, 0x1fff
; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v1, 2, 15
-; GFX9-GISEL-NEXT: s_and_b32 s6, s4, 0xffffe000
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x1ff0, v1
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s34
+; GFX9-GISEL-NEXT: s_mov_b32 s34, s32
+; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x4000
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, -16, v1
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_nop 1
@@ -1941,20 +1940,22 @@ define void @test_dynamic_stackalloc_device_divergent_over_aligned() #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s7, v0, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s6, v0, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v2, s7, 6, v1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x1bc
-; GFX9-GISEL-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
+; GFX9-GISEL-NEXT: s_add_u32 s5, s32, 0x1fff
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s6, 6
+; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xffffe000
+; GFX9-GISEL-NEXT: s_add_u32 s32, s5, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x1bc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-GISEL-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s34
-; GFX9-GISEL-NEXT: s_mov_b32 s34, s9
+; GFX9-GISEL-NEXT: s_mov_b32 s34, s8
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s8
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s7
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -2114,15 +2115,16 @@ define void @test_dynamic_stackalloc_device_divergent_under_aligned() #0 {
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_divergent_under_aligned:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s7, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s33
; GFX9-GISEL-NEXT: s_mov_b32 s33, s32
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v31
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v1, 2, 15
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x400
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x1ff0, v1
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v1, 2, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, -16, v1
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_nop 1
@@ -2137,20 +2139,19 @@ define void @test_dynamic_stackalloc_device_divergent_under_aligned() #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v0, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v0, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, s6, 6, v1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s33
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s7
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s8
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -2362,13 +2363,13 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) #0
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_multiple_allocas:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s11, s33
; GFX9-GISEL-NEXT: s_add_i32 s33, s32, 0xfc0
; GFX9-GISEL-NEXT: s_and_b32 s33, s33, 0xfffff000
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v2, off, s[0:3], s33 offset:64 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s34
+; GFX9-GISEL-NEXT: s_mov_b32 s12, s34
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: s_mov_b32 s34, s32
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x3000
@@ -2376,6 +2377,7 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) #0
; GFX9-GISEL-NEXT: s_cbranch_execz .LBB14_2
; GFX9-GISEL-NEXT: ; %bb.1: ; %bb.0
; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v1, 2, 15
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x3ff, v31
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, -16, v1
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[6:7]
@@ -2393,16 +2395,15 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) #0
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: v_readlane_b32 s8, v2, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-GISEL-NEXT: s_add_i32 s6, s32, 0xfff
-; GFX9-GISEL-NEXT: s_and_b32 s6, s6, 0xfffff000
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v3, s8, 6, v1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v3
-; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x3ff, v31
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v3, v3, 2, 15
-; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x1ff0, v3
+; GFX9-GISEL-NEXT: s_add_u32 s7, s32, 0xfff
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s8, 6
+; GFX9-GISEL-NEXT: s_and_b32 s8, s7, 0xfffff000
+; GFX9-GISEL-NEXT: s_add_u32 s32, s8, s6
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v3, 2, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s9, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, -16, v1
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v3, s[6:7]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[6:7]
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
@@ -2415,22 +2416,22 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) #0
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s8, v2, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s10, v2, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v3, s8, 6, v3
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v3
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 3
-; GFX9-GISEL-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s10, 6
+; GFX9-GISEL-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 4
-; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-GISEL-NEXT: s_add_u32 s32, s9, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: .LBB14_2: ; %bb.1
; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[4:5]
@@ -2446,23 +2447,23 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) #0
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v2, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v2, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s33
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s34
-; GFX9-GISEL-NEXT: s_mov_b32 s34, s10
+; GFX9-GISEL-NEXT: s_mov_b32 s34, s12
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:64 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s9
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s11
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -2789,13 +2790,13 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) #0 {
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_control_flow:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s10, s33
; GFX9-GISEL-NEXT: s_add_i32 s33, s32, 0xfc0
; GFX9-GISEL-NEXT: s_and_b32 s33, s33, 0xfffff000
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s34
+; GFX9-GISEL-NEXT: s_mov_b32 s11, s34
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: s_mov_b32 s34, s32
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x2000
@@ -2803,11 +2804,10 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) #0 {
; GFX9-GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execz .LBB15_2
; GFX9-GISEL-NEXT: ; %bb.1: ; %bb.1
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v1, v1, 2, 15
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 2
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, -16, v1
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v1, 2, 15
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[6:7]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
@@ -2822,11 +2822,12 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: v_readlane_b32 s8, v2, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-GISEL-NEXT: s_add_i32 s6, s32, 0xfff
-; GFX9-GISEL-NEXT: s_and_b32 s6, s6, 0xfffff000
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v3, s8, 6, v1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v3
+; GFX9-GISEL-NEXT: s_add_u32 s7, s32, 0xfff
+; GFX9-GISEL-NEXT: s_and_b32 s7, s7, 0xfffff000
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s8, 6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-GISEL-NEXT: s_add_u32 s32, s7, s6
; GFX9-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: ; implicit-def: $vgpr31
@@ -2836,7 +2837,8 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) #0 {
; GFX9-GISEL-NEXT: ; %bb.3: ; %bb.0
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_nop 1
@@ -2851,23 +2853,22 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) #0 {
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s8, v2, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s9, v2, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s8, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s9, 6
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 1
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s8
+; GFX9-GISEL-NEXT: s_add_u32 s32, s8, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: .LBB15_4: ; %bb.2
; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_mov_b32 s32, s34
-; GFX9-GISEL-NEXT: s_mov_b32 s34, s10
+; GFX9-GISEL-NEXT: s_mov_b32 s34, s11
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v2, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s9
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s10
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -3108,15 +3109,16 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_divergent_non_standard_size_i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s7, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s33
; GFX9-GISEL-NEXT: s_mov_b32 s33, s32
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x400
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff0, v0
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
; GFX9-GISEL-NEXT: s_nop 1
@@ -3131,20 +3133,19 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x29a
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s33
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s7
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s8
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -3290,13 +3291,14 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i64(i64
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_divergent_non_standard_size_i64:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s7, s33
+; GFX9-GISEL-NEXT: s_mov_b32 s8, s33
; GFX9-GISEL-NEXT: s_mov_b32 s33, s32
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-GISEL-NEXT: s_addk_i32 s32, 0x400
+; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, v0, 2, 15
+; GFX9-GISEL-NEXT: s_mov_b32 s6, s32
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
@@ -3312,20 +3314,19 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i64(i64
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-GISEL-NEXT: s_nop 1
; GFX9-GISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-GISEL-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-GISEL-NEXT: v_readlane_b32 s7, v1, 63
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s32
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s7, 6
+; GFX9-GISEL-NEXT: s_add_u32 s32, s6, s4
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x29a
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-GISEL-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_mov_b32 s32, s33
; GFX9-GISEL-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-GISEL-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-GISEL-NEXT: s_mov_b32 s33, s7
+; GFX9-GISEL-NEXT: s_mov_b32 s33, s8
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
index 2655165eceaf6..c544c9e0cee4e 100644
--- a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
@@ -85,23 +85,19 @@ define amdgpu_kernel void @fadd_f16(
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-GISEL-NEXT: s_mov_b32 s11, 0x31016000
; GFX11-GISEL-NEXT: s_mov_b32 s10, -1
-; GFX11-GISEL-NEXT: s_mov_b32 s7, s11
-; GFX11-GISEL-NEXT: s_mov_b32 s6, s10
+; GFX11-GISEL-NEXT: s_mov_b32 s11, 0x31016000
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_mov_b64 s[6:7], s[10:11]
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_mov_b32 s8, s0
-; GFX11-GISEL-NEXT: s_mov_b32 s9, s1
-; GFX11-GISEL-NEXT: s_mov_b32 s0, s2
-; GFX11-GISEL-NEXT: s_mov_b32 s1, s3
-; GFX11-GISEL-NEXT: s_mov_b32 s2, s10
-; GFX11-GISEL-NEXT: s_mov_b32 s3, s11
-; GFX11-GISEL-NEXT: buffer_load_d16_b16 v0, off, s[0:3], 0 glc dlc
+; GFX11-GISEL-NEXT: s_mov_b64 s[8:9], s[2:3]
+; GFX11-GISEL-NEXT: s_mov_b64 s[2:3], s[10:11]
+; GFX11-GISEL-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0 glc dlc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], 0 glc dlc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-GISEL-NEXT: buffer_store_b16 v0, off, s[8:11], 0
+; GFX11-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-NEXT: s_endpgm
;
; GFX11-FAKE16-SDAG-LABEL: fadd_f16:
@@ -133,23 +129,22 @@ define amdgpu_kernel void @fadd_f16(
; GFX11-FAKE16-GISEL-NEXT: s_clause 0x1
; GFX11-FAKE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s11, 0x31016000
; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s10, -1
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s7, s11
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s6, s10
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s11, 0x31016000
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[6:7], s[10:11]
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s8, s0
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s9, s1
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s0, s2
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s1, s3
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s2, s10
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s3, s11
-; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[0:3], 0 glc dlc
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[8:9], s[2:3]
+; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[8:11], 0 glc dlc
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v1, off, s[4:7], 0 glc dlc
+; GFX11-FAKE16-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[4:7], 0 glc dlc
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[8:11], 0
+; GFX11-FAKE16-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_add_f16_e64 v0, s2, s3
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[2:3], s[10:11]
+; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
; GFX11-LABEL: fadd_f16:
; GFX11: ; %bb.0: ; %entry
@@ -249,19 +244,15 @@ define amdgpu_kernel void @fadd_f16_imm_a(
; GFX11-GISEL-LABEL: fadd_f16_imm_a:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX11-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-NEXT: s_mov_b32 s0, s2
-; GFX11-GISEL-NEXT: s_mov_b32 s1, s3
-; GFX11-GISEL-NEXT: s_mov_b32 s2, s6
-; GFX11-GISEL-NEXT: s_mov_b32 s3, s7
-; GFX11-GISEL-NEXT: buffer_load_d16_b16 v0, off, s[0:3], 0
+; GFX11-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-GISEL-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
-; GFX11-GISEL-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-NEXT: s_endpgm
;
; GFX11-FAKE16-SDAG-LABEL: fadd_f16_imm_a:
@@ -285,19 +276,17 @@ define amdgpu_kernel void @fadd_f16_imm_a(
; GFX11-FAKE16-GISEL-LABEL: fadd_f16_imm_a:
; GFX11-FAKE16-GISEL: ; %bb.0: ; %entry
; GFX11-FAKE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s0, s2
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s1, s3
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s2, s6
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s3, s7
-; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[0:3], 0
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[4:7], 0
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-GISEL-NEXT: v_add_f16_e32 v0, 1.0, v0
-; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-FAKE16-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_add_f16_e64 v0, s2, 1.0
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
; GFX11-LABEL: fadd_f16_imm_a:
; GFX11: ; %bb.0: ; %entry
@@ -389,19 +378,15 @@ define amdgpu_kernel void @fadd_f16_imm_b(
; GFX11-GISEL-LABEL: fadd_f16_imm_b:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX11-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-NEXT: s_mov_b32 s0, s2
-; GFX11-GISEL-NEXT: s_mov_b32 s1, s3
-; GFX11-GISEL-NEXT: s_mov_b32 s2, s6
-; GFX11-GISEL-NEXT: s_mov_b32 s3, s7
-; GFX11-GISEL-NEXT: buffer_load_d16_b16 v0, off, s[0:3], 0
+; GFX11-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-GISEL-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, 2.0, v0.l
-; GFX11-GISEL-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-NEXT: s_endpgm
;
; GFX11-FAKE16-SDAG-LABEL: fadd_f16_imm_b:
@@ -425,19 +410,17 @@ define amdgpu_kernel void @fadd_f16_imm_b(
; GFX11-FAKE16-GISEL-LABEL: fadd_f16_imm_b:
; GFX11-FAKE16-GISEL: ; %bb.0: ; %entry
; GFX11-FAKE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s0, s2
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s1, s3
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s2, s6
-; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s3, s7
-; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[0:3], 0
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[4:7], 0
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-GISEL-NEXT: v_add_f16_e32 v0, 2.0, v0
-; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-FAKE16-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_add_f16_e64 v0, s2, 2.0
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-GISEL-NEXT: s_endpgm
; GFX11-LABEL: fadd_f16_imm_b:
; GFX11: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index de89b4912ad4e..2533c25a86edf 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -3275,405 +3275,1678 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
}
define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_and_saddr_i64_rtn:
+; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB34_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB34_4
+; GFX1250-SDAG-NEXT: .LBB34_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB34_5
+; GFX1250-SDAG-NEXT: .LBB34_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB34_2
+; GFX1250-SDAG-NEXT: .LBB34_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v3, v1, v3
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB34_5
+; GFX1250-SDAG-NEXT: .LBB34_5:
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB34_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB34_4
+; GFX1250-GISEL-NEXT: .LBB34_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB34_5
+; GFX1250-GISEL-NEXT: .LBB34_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB34_2
+; GFX1250-GISEL-NEXT: .LBB34_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v2, v0, v4
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v3, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB34_5
+; GFX1250-GISEL-NEXT: .LBB34_5:
+;
+; GFX950-SDAG-LABEL: flat_and_saddr_i64_rtn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB34_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB34_4
+; GFX950-SDAG-NEXT: .LBB34_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_branch .LBB34_5
+; GFX950-SDAG-NEXT: .LBB34_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB34_2
+; GFX950-SDAG-NEXT: .LBB34_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_and_b32_e32 v3, v1, v3
+; GFX950-SDAG-NEXT: v_and_b32_e32 v2, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: s_branch .LBB34_5
+; GFX950-SDAG-NEXT: .LBB34_5:
+;
+; GFX950-GISEL-LABEL: flat_and_saddr_i64_rtn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB34_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB34_4
+; GFX950-GISEL-NEXT: .LBB34_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_branch .LBB34_5
+; GFX950-GISEL-NEXT: .LBB34_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB34_2
+; GFX950-GISEL-NEXT: .LBB34_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_and_b32_e32 v2, v0, v4
+; GFX950-GISEL-NEXT: v_and_b32_e32 v3, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: s_branch .LBB34_5
+; GFX950-GISEL-NEXT: .LBB34_5:
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %rtn = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst
+ %cast.rtn = bitcast i64 %rtn to <2 x float>
+ ret <2 x float> %cast.rtn
+}
+
+define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB35_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB35_4
+; GFX1250-SDAG-NEXT: .LBB35_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB35_5
+; GFX1250-SDAG-NEXT: .LBB35_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB35_2
+; GFX1250-SDAG-NEXT: .LBB35_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v3, v1, v3
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB35_5
+; GFX1250-SDAG-NEXT: .LBB35_5:
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB35_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB35_4
+; GFX1250-GISEL-NEXT: .LBB35_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB35_5
+; GFX1250-GISEL-NEXT: .LBB35_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB35_2
+; GFX1250-GISEL-NEXT: .LBB35_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v2, v0, v4
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v3, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB35_5
+; GFX1250-GISEL-NEXT: .LBB35_5:
+;
+; GFX950-SDAG-LABEL: flat_and_saddr_i64_rtn_neg128:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff80
+; GFX950-SDAG-NEXT: s_mov_b32 s3, -1
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB35_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB35_4
+; GFX950-SDAG-NEXT: .LBB35_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_branch .LBB35_5
+; GFX950-SDAG-NEXT: .LBB35_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB35_2
+; GFX950-SDAG-NEXT: .LBB35_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_and_b32_e32 v3, v1, v3
+; GFX950-SDAG-NEXT: v_and_b32_e32 v2, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: s_branch .LBB35_5
+; GFX950-SDAG-NEXT: .LBB35_5:
+;
+; GFX950-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB35_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB35_4
+; GFX950-GISEL-NEXT: .LBB35_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_branch .LBB35_5
+; GFX950-GISEL-NEXT: .LBB35_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB35_2
+; GFX950-GISEL-NEXT: .LBB35_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_and_b32_e32 v2, v0, v4
+; GFX950-GISEL-NEXT: v_and_b32_e32 v3, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: s_branch .LBB35_5
+; GFX950-GISEL-NEXT: .LBB35_5:
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
+ %rtn = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst
+ %cast.rtn = bitcast i64 %rtn to <2 x float>
+ ret <2 x float> %cast.rtn
+}
+
+define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB36_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB36_4
+; GFX1250-SDAG-NEXT: .LBB36_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-NEXT: .LBB36_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB36_2
+; GFX1250-SDAG-NEXT: .LBB36_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB36_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB36_4
+; GFX1250-GISEL-NEXT: .LBB36_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-GISEL-NEXT: .LBB36_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB36_2
+; GFX1250-GISEL-NEXT: .LBB36_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
+; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX950-SDAG-LABEL: flat_and_saddr_i64_nortn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB36_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB36_4
+; GFX950-SDAG-NEXT: .LBB36_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_endpgm
+; GFX950-SDAG-NEXT: .LBB36_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB36_2
+; GFX950-SDAG-NEXT: .LBB36_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX950-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: flat_and_saddr_i64_nortn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB36_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB36_4
+; GFX950-GISEL-NEXT: .LBB36_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_endpgm
+; GFX950-GISEL-NEXT: .LBB36_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB36_2
+; GFX950-GISEL-NEXT: .LBB36_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off
+; GFX950-GISEL-NEXT: s_endpgm
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %unused = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst
+ ret void
+}
+
+define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
+; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB37_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB37_4
+; GFX1250-SDAG-NEXT: .LBB37_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-NEXT: .LBB37_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB37_2
+; GFX1250-SDAG-NEXT: .LBB37_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB37_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB37_4
+; GFX1250-GISEL-NEXT: .LBB37_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-GISEL-NEXT: .LBB37_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB37_2
+; GFX1250-GISEL-NEXT: .LBB37_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
+; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX950-SDAG-LABEL: flat_and_saddr_i64_nortn_neg128:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff80
+; GFX950-SDAG-NEXT: s_mov_b32 s3, -1
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB37_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB37_4
+; GFX950-SDAG-NEXT: .LBB37_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_endpgm
+; GFX950-SDAG-NEXT: .LBB37_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB37_2
+; GFX950-SDAG-NEXT: .LBB37_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX950-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB37_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB37_4
+; GFX950-GISEL-NEXT: .LBB37_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_endpgm
+; GFX950-GISEL-NEXT: .LBB37_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB37_2
+; GFX950-GISEL-NEXT: .LBB37_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off
+; GFX950-GISEL-NEXT: s_endpgm
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
+ %unused = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst
+ ret void
+}
+
+; --------------------------------------------------------------------------------
+; atomicrmw or
+; --------------------------------------------------------------------------------
+
+define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_or_saddr_i32_rtn:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
+;
+; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or v0, v[0:1], v2 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or v0, v[2:3], v1 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %rtn = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst
+ %cast.rtn = bitcast i32 %rtn to float
+ ret float %cast.rtn
+}
+
+define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_or_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
-; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB34_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB34_4
-; GFX1250-NEXT: .LBB34_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB34_5
-; GFX1250-NEXT: .LBB34_3: ; %atomicrmw.global
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB34_2
-; GFX1250-NEXT: .LBB34_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-NEXT: ; return to shader part epilog
+;
+; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn_neg128:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or v0, v[0:1], v2 sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn_neg128:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or v0, v[2:3], v1 sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
+ %rtn = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst
+ %cast.rtn = bitcast i32 %rtn to float
+ ret float %cast.rtn
+}
+
+define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_or_saddr_i32_nortn:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v3, v1, v3
-; GFX1250-NEXT: v_and_b32_e32 v2, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB34_5
-; GFX1250-NEXT: .LBB34_5:
+; GFX1250-NEXT: s_endpgm
+;
+; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: s_endpgm
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %unused = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst
+ ret void
+}
+
+define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_or_saddr_i32_nortn_neg128:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_endpgm
+;
+; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn_neg128:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn_neg128:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: s_endpgm
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
+ %unused = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst
+ ret void
+}
+
+define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB42_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB42_4
+; GFX1250-SDAG-NEXT: .LBB42_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB42_5
+; GFX1250-SDAG-NEXT: .LBB42_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB42_2
+; GFX1250-SDAG-NEXT: .LBB42_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB42_5
+; GFX1250-SDAG-NEXT: .LBB42_5:
+;
+; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB42_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB42_4
+; GFX1250-GISEL-NEXT: .LBB42_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB42_5
+; GFX1250-GISEL-NEXT: .LBB42_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB42_2
+; GFX1250-GISEL-NEXT: .LBB42_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v2, v0, v4
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v3, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB42_5
+; GFX1250-GISEL-NEXT: .LBB42_5:
+;
+; GFX950-SDAG-LABEL: flat_or_saddr_i64_rtn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB42_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB42_4
+; GFX950-SDAG-NEXT: .LBB42_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_branch .LBB42_5
+; GFX950-SDAG-NEXT: .LBB42_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB42_2
+; GFX950-SDAG-NEXT: .LBB42_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX950-SDAG-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: s_branch .LBB42_5
+; GFX950-SDAG-NEXT: .LBB42_5:
;
-; GFX950-LABEL: flat_and_saddr_i64_rtn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB34_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB34_4
-; GFX950-NEXT: .LBB34_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_branch .LBB34_5
-; GFX950-NEXT: .LBB34_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX950-NEXT: ; implicit-def: $vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB34_2
-; GFX950-NEXT: .LBB34_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, v1, v3
-; GFX950-NEXT: v_and_b32_e32 v2, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: s_branch .LBB34_5
-; GFX950-NEXT: .LBB34_5:
+; GFX950-GISEL-LABEL: flat_or_saddr_i64_rtn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB42_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB42_4
+; GFX950-GISEL-NEXT: .LBB42_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_branch .LBB42_5
+; GFX950-GISEL-NEXT: .LBB42_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB42_2
+; GFX950-GISEL-NEXT: .LBB42_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_or_b32_e32 v2, v0, v4
+; GFX950-GISEL-NEXT: v_or_b32_e32 v3, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: s_branch .LBB42_5
+; GFX950-GISEL-NEXT: .LBB42_5:
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %rtn = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst
+ %rtn = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst
%cast.rtn = bitcast i64 %rtn to <2 x float>
ret <2 x float> %cast.rtn
}
-define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_and_saddr_i64_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB35_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB35_4
-; GFX1250-NEXT: .LBB35_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB35_5
-; GFX1250-NEXT: .LBB35_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB35_2
-; GFX1250-NEXT: .LBB35_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v3, v1, v3
-; GFX1250-NEXT: v_and_b32_e32 v2, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB35_5
-; GFX1250-NEXT: .LBB35_5:
+define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB43_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB43_4
+; GFX1250-SDAG-NEXT: .LBB43_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB43_5
+; GFX1250-SDAG-NEXT: .LBB43_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB43_2
+; GFX1250-SDAG-NEXT: .LBB43_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB43_5
+; GFX1250-SDAG-NEXT: .LBB43_5:
;
-; GFX950-LABEL: flat_and_saddr_i64_rtn_neg128:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: s_movk_i32 s2, 0xff80
-; GFX950-NEXT: s_mov_b32 s3, -1
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB35_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB35_4
-; GFX950-NEXT: .LBB35_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_branch .LBB35_5
-; GFX950-NEXT: .LBB35_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX950-NEXT: ; implicit-def: $vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB35_2
-; GFX950-NEXT: .LBB35_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, v1, v3
-; GFX950-NEXT: v_and_b32_e32 v2, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: s_branch .LBB35_5
-; GFX950-NEXT: .LBB35_5:
+; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB43_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB43_4
+; GFX1250-GISEL-NEXT: .LBB43_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB43_5
+; GFX1250-GISEL-NEXT: .LBB43_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB43_2
+; GFX1250-GISEL-NEXT: .LBB43_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v2, v0, v4
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v3, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB43_5
+; GFX1250-GISEL-NEXT: .LBB43_5:
+;
+; GFX950-SDAG-LABEL: flat_or_saddr_i64_rtn_neg128:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff80
+; GFX950-SDAG-NEXT: s_mov_b32 s3, -1
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB43_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB43_4
+; GFX950-SDAG-NEXT: .LBB43_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_branch .LBB43_5
+; GFX950-SDAG-NEXT: .LBB43_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB43_2
+; GFX950-SDAG-NEXT: .LBB43_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX950-SDAG-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: s_branch .LBB43_5
+; GFX950-SDAG-NEXT: .LBB43_5:
+;
+; GFX950-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB43_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB43_4
+; GFX950-GISEL-NEXT: .LBB43_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_branch .LBB43_5
+; GFX950-GISEL-NEXT: .LBB43_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB43_2
+; GFX950-GISEL-NEXT: .LBB43_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_or_b32_e32 v2, v0, v4
+; GFX950-GISEL-NEXT: v_or_b32_e32 v3, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: s_branch .LBB43_5
+; GFX950-GISEL-NEXT: .LBB43_5:
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %rtn = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst
+ %rtn = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst
%cast.rtn = bitcast i64 %rtn to <2 x float>
ret <2 x float> %cast.rtn
}
-define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_and_saddr_i64_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB36_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB36_4
-; GFX1250-NEXT: .LBB36_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_endpgm
-; GFX1250-NEXT: .LBB36_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB36_2
-; GFX1250-NEXT: .LBB36_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX1250-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[0:1], off
-; GFX1250-NEXT: s_endpgm
+define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB44_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB44_4
+; GFX1250-SDAG-NEXT: .LBB44_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-NEXT: .LBB44_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB44_2
+; GFX1250-SDAG-NEXT: .LBB44_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB44_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB44_4
+; GFX1250-GISEL-NEXT: .LBB44_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-GISEL-NEXT: .LBB44_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB44_2
+; GFX1250-GISEL-NEXT: .LBB44_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
+; GFX1250-GISEL-NEXT: s_endpgm
;
-; GFX950-LABEL: flat_and_saddr_i64_nortn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB36_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB36_4
-; GFX950-NEXT: .LBB36_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_endpgm
-; GFX950-NEXT: .LBB36_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB36_2
-; GFX950-NEXT: .LBB36_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX950-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off
-; GFX950-NEXT: s_endpgm
+; GFX950-SDAG-LABEL: flat_or_saddr_i64_nortn:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB44_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB44_4
+; GFX950-SDAG-NEXT: .LBB44_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_endpgm
+; GFX950-SDAG-NEXT: .LBB44_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB44_2
+; GFX950-SDAG-NEXT: .LBB44_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX950-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: flat_or_saddr_i64_nortn:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB44_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB44_4
+; GFX950-GISEL-NEXT: .LBB44_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_endpgm
+; GFX950-GISEL-NEXT: .LBB44_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB44_2
+; GFX950-GISEL-NEXT: .LBB44_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off
+; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %unused = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst
+ %unused = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst
ret void
}
-define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_and_saddr_i64_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB37_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB37_4
-; GFX1250-NEXT: .LBB37_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_endpgm
-; GFX1250-NEXT: .LBB37_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB37_2
-; GFX1250-NEXT: .LBB37_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX1250-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[0:1], off
-; GFX1250-NEXT: s_endpgm
+define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
+; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB45_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB45_4
+; GFX1250-SDAG-NEXT: .LBB45_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-NEXT: .LBB45_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB45_2
+; GFX1250-SDAG-NEXT: .LBB45_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX1250-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX1250-SDAG-NEXT: s_endpgm
;
-; GFX950-LABEL: flat_and_saddr_i64_nortn_neg128:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: s_movk_i32 s2, 0xff80
-; GFX950-NEXT: s_mov_b32 s3, -1
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB37_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB37_4
-; GFX950-NEXT: .LBB37_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_endpgm
-; GFX950-NEXT: .LBB37_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB37_2
-; GFX950-NEXT: .LBB37_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX950-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off
-; GFX950-NEXT: s_endpgm
+; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB45_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB45_4
+; GFX1250-GISEL-NEXT: .LBB45_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-GISEL-NEXT: .LBB45_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB45_2
+; GFX1250-GISEL-NEXT: .LBB45_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX1250-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
+; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX950-SDAG-LABEL: flat_or_saddr_i64_nortn_neg128:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff80
+; GFX950-SDAG-NEXT: s_mov_b32 s3, -1
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB45_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB45_4
+; GFX950-SDAG-NEXT: .LBB45_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_endpgm
+; GFX950-SDAG-NEXT: .LBB45_3: ; %atomicrmw.global
+; GFX950-SDAG-NEXT: buffer_wbl2 sc1
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB45_2
+; GFX950-SDAG-NEXT: .LBB45_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX950-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB45_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB45_4
+; GFX950-GISEL-NEXT: .LBB45_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_endpgm
+; GFX950-GISEL-NEXT: .LBB45_3: ; %atomicrmw.global
+; GFX950-GISEL-NEXT: buffer_wbl2 sc1
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB45_2
+; GFX950-GISEL-NEXT: .LBB45_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off
+; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %unused = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst
+ %unused = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst
ret void
}
; --------------------------------------------------------------------------------
-; atomicrmw or
+; atomicrmw xor
; --------------------------------------------------------------------------------
-define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_rtn:
+define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_xor_saddr_i32_rtn:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
;
-; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn:
+; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_or v0, v[0:1], v2 sc0
+; GFX950-SDAG-NEXT: flat_atomic_xor v0, v[0:1], v2 sc0
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; return to shader part epilog
;
-; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
@@ -3681,30 +4954,30 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i3
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_or v0, v[2:3], v1 sc0
+; GFX950-GISEL-NEXT: flat_atomic_xor v0, v[2:3], v1 sc0
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: buffer_inv sc1
; GFX950-GISEL-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %rtn = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst
+ %rtn = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst
%cast.rtn = bitcast i32 %rtn to float
ret float %cast.rtn
}
-define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_rtn_neg128:
+define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_xor_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
;
-; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn_neg128:
+; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -3714,12 +4987,12 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voff
; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_or v0, v[0:1], v2 sc0
+; GFX950-SDAG-NEXT: flat_atomic_xor v0, v[0:1], v2 sc0
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; return to shader part epilog
;
-; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn_neg128:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
@@ -3730,43 +5003,43 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voff
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_or v0, v[2:3], v1 sc0
+; GFX950-GISEL-NEXT: flat_atomic_xor v0, v[2:3], v1 sc0
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: buffer_inv sc1
; GFX950-GISEL-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %rtn = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst
+ %rtn = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst
%cast.rtn = bitcast i32 %rtn to float
ret float %cast.rtn
}
-define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_nortn:
+define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_xor_saddr_i32_nortn:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_endpgm
;
-; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn:
+; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_or v[0:1], v2
+; GFX950-SDAG-NEXT: flat_atomic_xor v[0:1], v2
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: s_endpgm
;
-; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
@@ -3774,29 +5047,29 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_or v[2:3], v1
+; GFX950-GISEL-NEXT: flat_atomic_xor v[2:3], v1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: buffer_inv sc1
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %unused = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst
+ %unused = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst
ret void
}
-define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_nortn_neg128:
+define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
+; GFX1250-LABEL: flat_xor_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_endpgm
;
-; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn_neg128:
+; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -3806,12 +5079,12 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vof
; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_or v[0:1], v2
+; GFX950-SDAG-NEXT: flat_atomic_xor v[0:1], v2
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: s_endpgm
;
-; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn_neg128:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
@@ -3822,940 +5095,738 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vof
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_or v[2:3], v1
+; GFX950-GISEL-NEXT: flat_atomic_xor v[2:3], v1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: buffer_inv sc1
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %unused = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst
- ret void
-}
-
-define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_or_saddr_i64_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
-; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB42_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB42_4
-; GFX1250-NEXT: .LBB42_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB42_5
-; GFX1250-NEXT: .LBB42_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB42_2
-; GFX1250-NEXT: .LBB42_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v3, v1, v3
-; GFX1250-NEXT: v_or_b32_e32 v2, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB42_5
-; GFX1250-NEXT: .LBB42_5:
-;
-; GFX950-LABEL: flat_or_saddr_i64_rtn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB42_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB42_4
-; GFX950-NEXT: .LBB42_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_branch .LBB42_5
-; GFX950-NEXT: .LBB42_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX950-NEXT: ; implicit-def: $vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB42_2
-; GFX950-NEXT: .LBB42_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_or_b32_e32 v3, v1, v3
-; GFX950-NEXT: v_or_b32_e32 v2, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: s_branch .LBB42_5
-; GFX950-NEXT: .LBB42_5:
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %rtn = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst
- %cast.rtn = bitcast i64 %rtn to <2 x float>
- ret <2 x float> %cast.rtn
-}
-
-define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_or_saddr_i64_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB43_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB43_4
-; GFX1250-NEXT: .LBB43_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB43_5
-; GFX1250-NEXT: .LBB43_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB43_2
-; GFX1250-NEXT: .LBB43_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v3, v1, v3
-; GFX1250-NEXT: v_or_b32_e32 v2, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB43_5
-; GFX1250-NEXT: .LBB43_5:
-;
-; GFX950-LABEL: flat_or_saddr_i64_rtn_neg128:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: s_movk_i32 s2, 0xff80
-; GFX950-NEXT: s_mov_b32 s3, -1
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB43_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB43_4
-; GFX950-NEXT: .LBB43_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_branch .LBB43_5
-; GFX950-NEXT: .LBB43_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX950-NEXT: ; implicit-def: $vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB43_2
-; GFX950-NEXT: .LBB43_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_or_b32_e32 v3, v1, v3
-; GFX950-NEXT: v_or_b32_e32 v2, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: s_branch .LBB43_5
-; GFX950-NEXT: .LBB43_5:
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %rtn = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst
- %cast.rtn = bitcast i64 %rtn to <2 x float>
- ret <2 x float> %cast.rtn
-}
-
-define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_or_saddr_i64_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB44_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB44_4
-; GFX1250-NEXT: .LBB44_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_endpgm
-; GFX1250-NEXT: .LBB44_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB44_2
-; GFX1250-NEXT: .LBB44_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX1250-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[0:1], off
-; GFX1250-NEXT: s_endpgm
-;
-; GFX950-LABEL: flat_or_saddr_i64_nortn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB44_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB44_4
-; GFX950-NEXT: .LBB44_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_endpgm
-; GFX950-NEXT: .LBB44_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB44_2
-; GFX950-NEXT: .LBB44_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX950-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off
-; GFX950-NEXT: s_endpgm
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %unused = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst
+ %unused = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst
ret void
}
-define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_or_saddr_i64_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB45_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB45_4
-; GFX1250-NEXT: .LBB45_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_endpgm
-; GFX1250-NEXT: .LBB45_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB45_2
-; GFX1250-NEXT: .LBB45_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX1250-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[0:1], off
-; GFX1250-NEXT: s_endpgm
+define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB50_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB50_4
+; GFX1250-SDAG-NEXT: .LBB50_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB50_5
+; GFX1250-SDAG-NEXT: .LBB50_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB50_2
+; GFX1250-SDAG-NEXT: .LBB50_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v3, v1, v3
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB50_5
+; GFX1250-SDAG-NEXT: .LBB50_5:
;
-; GFX950-LABEL: flat_or_saddr_i64_nortn_neg128:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: s_movk_i32 s2, 0xff80
-; GFX950-NEXT: s_mov_b32 s3, -1
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB45_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB45_4
-; GFX950-NEXT: .LBB45_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_endpgm
-; GFX950-NEXT: .LBB45_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB45_2
-; GFX950-NEXT: .LBB45_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX950-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off
-; GFX950-NEXT: s_endpgm
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %unused = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst
- ret void
-}
-
-; --------------------------------------------------------------------------------
-; atomicrmw xor
-; --------------------------------------------------------------------------------
-
-define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB50_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB50_4
+; GFX1250-GISEL-NEXT: .LBB50_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB50_5
+; GFX1250-GISEL-NEXT: .LBB50_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB50_2
+; GFX1250-GISEL-NEXT: .LBB50_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, v0, v4
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v3, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB50_5
+; GFX1250-GISEL-NEXT: .LBB50_5:
;
-; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn:
+; GFX950-SDAG-LABEL: flat_xor_saddr_i64_rtn:
; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB50_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB50_4
+; GFX950-SDAG-NEXT: .LBB50_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_branch .LBB50_5
+; GFX950-SDAG-NEXT: .LBB50_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_xor v0, v[0:1], v2 sc0
+; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] sc0
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
-; GFX950-SDAG-NEXT: ; return to shader part epilog
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB50_2
+; GFX950-SDAG-NEXT: .LBB50_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v3, v1, v3
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v2, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: s_branch .LBB50_5
+; GFX950-SDAG-NEXT: .LBB50_5:
;
-; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
-; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB50_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB50_4
+; GFX950-GISEL-NEXT: .LBB50_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_branch .LBB50_5
+; GFX950-GISEL-NEXT: .LBB50_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_xor v0, v[2:3], v1 sc0
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-GISEL-NEXT: buffer_inv sc1
-; GFX950-GISEL-NEXT: ; return to shader part epilog
+; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB50_2
+; GFX950-GISEL-NEXT: .LBB50_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v2, v0, v4
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v3, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: s_branch .LBB50_5
+; GFX950-GISEL-NEXT: .LBB50_5:
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %rtn = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst
- %cast.rtn = bitcast i32 %rtn to float
- ret float %cast.rtn
+ %rtn = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst
+ %cast.rtn = bitcast i64 %rtn to <2 x float>
+ ret <2 x float> %cast.rtn
}
-define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB51_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB51_4
+; GFX1250-SDAG-NEXT: .LBB51_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB51_5
+; GFX1250-SDAG-NEXT: .LBB51_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB51_2
+; GFX1250-SDAG-NEXT: .LBB51_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v3, v1, v3
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_branch .LBB51_5
+; GFX1250-SDAG-NEXT: .LBB51_5:
;
-; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn_neg128:
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB51_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB51_4
+; GFX1250-GISEL-NEXT: .LBB51_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB51_5
+; GFX1250-GISEL-NEXT: .LBB51_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB51_2
+; GFX1250-GISEL-NEXT: .LBB51_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, v0, v4
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v3, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_branch .LBB51_5
+; GFX1250-GISEL-NEXT: .LBB51_5:
+;
+; GFX950-SDAG-LABEL: flat_xor_saddr_i64_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-SDAG-NEXT: s_nop 1
-; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff80
+; GFX950-SDAG-NEXT: s_mov_b32 s3, -1
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB51_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB51_4
+; GFX950-SDAG-NEXT: .LBB51_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_branch .LBB51_5
+; GFX950-SDAG-NEXT: .LBB51_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_xor v0, v[0:1], v2 sc0
+; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] sc0
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
-; GFX950-SDAG-NEXT: ; return to shader part epilog
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB51_2
+; GFX950-SDAG-NEXT: .LBB51_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v3, v1, v3
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v2, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off
+; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: s_branch .LBB51_5
+; GFX950-SDAG-NEXT: .LBB51_5:
;
-; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn_neg128:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
-; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB51_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB51_4
+; GFX950-GISEL-NEXT: .LBB51_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_branch .LBB51_5
+; GFX950-GISEL-NEXT: .LBB51_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_xor v0, v[2:3], v1 sc0
+; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[4:5] sc0
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: buffer_inv sc1
-; GFX950-GISEL-NEXT: ; return to shader part epilog
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB51_2
+; GFX950-GISEL-NEXT: .LBB51_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v2, v0, v4
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v3, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off
+; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: s_branch .LBB51_5
+; GFX950-GISEL-NEXT: .LBB51_5:
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %rtn = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst
- %cast.rtn = bitcast i32 %rtn to float
- ret float %cast.rtn
+ %rtn = atomicrmw xor ptr %gep1, i64 %data syncscope("agent") seq_cst
+ %cast.rtn = bitcast i64 %rtn to <2 x float>
+ ret <2 x float> %cast.rtn
}
-define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB52_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB52_4
+; GFX1250-SDAG-NEXT: .LBB52_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-NEXT: .LBB52_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB52_2
+; GFX1250-SDAG-NEXT: .LBB52_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX1250-SDAG-NEXT: s_endpgm
;
-; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn:
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB52_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB52_4
+; GFX1250-GISEL-NEXT: .LBB52_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-GISEL-NEXT: .LBB52_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB52_2
+; GFX1250-GISEL-NEXT: .LBB52_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
+; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX950-SDAG-LABEL: flat_xor_saddr_i64_nortn:
; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB52_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB52_4
+; GFX950-SDAG-NEXT: .LBB52_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_endpgm
+; GFX950-SDAG-NEXT: .LBB52_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_xor v[0:1], v2
+; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB52_2
+; GFX950-SDAG-NEXT: .LBB52_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off
; GFX950-SDAG-NEXT: s_endpgm
;
-; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
-; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB52_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB52_4
+; GFX950-GISEL-NEXT: .LBB52_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_endpgm
+; GFX950-GISEL-NEXT: .LBB52_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_xor v[2:3], v1
+; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5]
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB52_2
+; GFX950-GISEL-NEXT: .LBB52_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %unused = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst
+ %unused = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst
ret void
}
-define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
+; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
+; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB53_3
+; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB53_4
+; GFX1250-SDAG-NEXT: .LBB53_2: ; %atomicrmw.phi
+; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-NEXT: .LBB53_3: ; %atomicrmw.global
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3
+; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB53_2
+; GFX1250-SDAG-NEXT: .LBB53_4: ; %atomicrmw.private
+; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
+; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX1250-SDAG-NEXT: s_endpgm
;
-; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn_neg128:
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB53_3
+; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB53_4
+; GFX1250-GISEL-NEXT: .LBB53_2: ; %atomicrmw.phi
+; GFX1250-GISEL-NEXT: s_endpgm
+; GFX1250-GISEL-NEXT: .LBB53_3: ; %atomicrmw.global
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB53_2
+; GFX1250-GISEL-NEXT: .LBB53_4: ; %atomicrmw.private
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
+; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX950-SDAG-LABEL: flat_xor_saddr_i64_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-SDAG-NEXT: s_nop 1
-; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff80
+; GFX950-SDAG-NEXT: s_mov_b32 s3, -1
+; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB53_3
+; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB53_4
+; GFX950-SDAG-NEXT: .LBB53_2: ; %atomicrmw.phi
+; GFX950-SDAG-NEXT: s_endpgm
+; GFX950-SDAG-NEXT: .LBB53_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: buffer_wbl2 sc1
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: flat_atomic_xor v[0:1], v2
+; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-SDAG-NEXT: s_cbranch_execz .LBB53_2
+; GFX950-SDAG-NEXT: .LBB53_4: ; %atomicrmw.private
+; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-SDAG-NEXT: s_nop 1
+; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX950-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off
; GFX950-SDAG-NEXT: s_endpgm
;
-; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn_neg128:
+; GFX950-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
-; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
+; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB53_3
+; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB53_4
+; GFX950-GISEL-NEXT: .LBB53_2: ; %atomicrmw.phi
+; GFX950-GISEL-NEXT: s_endpgm
+; GFX950-GISEL-NEXT: .LBB53_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: flat_atomic_xor v[2:3], v1
+; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5]
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: buffer_inv sc1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4
+; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX950-GISEL-NEXT: s_cbranch_execz .LBB53_2
+; GFX950-GISEL-NEXT: .LBB53_4: ; %atomicrmw.private
+; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off
; GFX950-GISEL-NEXT: s_endpgm
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %unused = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst
- ret void
-}
-
-define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i64_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
-; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB50_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB50_4
-; GFX1250-NEXT: .LBB50_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB50_5
-; GFX1250-NEXT: .LBB50_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB50_2
-; GFX1250-NEXT: .LBB50_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v3, v1, v3
-; GFX1250-NEXT: v_xor_b32_e32 v2, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB50_5
-; GFX1250-NEXT: .LBB50_5:
-;
-; GFX950-LABEL: flat_xor_saddr_i64_rtn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB50_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB50_4
-; GFX950-NEXT: .LBB50_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_branch .LBB50_5
-; GFX950-NEXT: .LBB50_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX950-NEXT: ; implicit-def: $vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB50_2
-; GFX950-NEXT: .LBB50_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_xor_b32_e32 v3, v1, v3
-; GFX950-NEXT: v_xor_b32_e32 v2, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: s_branch .LBB50_5
-; GFX950-NEXT: .LBB50_5:
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %rtn = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst
- %cast.rtn = bitcast i64 %rtn to <2 x float>
- ret <2 x float> %cast.rtn
-}
-
-define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i64_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB51_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB51_4
-; GFX1250-NEXT: .LBB51_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB51_5
-; GFX1250-NEXT: .LBB51_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB51_2
-; GFX1250-NEXT: .LBB51_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v3, v1, v3
-; GFX1250-NEXT: v_xor_b32_e32 v2, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_branch .LBB51_5
-; GFX1250-NEXT: .LBB51_5:
-;
-; GFX950-LABEL: flat_xor_saddr_i64_rtn_neg128:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: s_movk_i32 s2, 0xff80
-; GFX950-NEXT: s_mov_b32 s3, -1
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB51_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB51_4
-; GFX950-NEXT: .LBB51_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_branch .LBB51_5
-; GFX950-NEXT: .LBB51_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] sc0
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX950-NEXT: ; implicit-def: $vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB51_2
-; GFX950-NEXT: .LBB51_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_xor_b32_e32 v3, v1, v3
-; GFX950-NEXT: v_xor_b32_e32 v2, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off
-; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: s_branch .LBB51_5
-; GFX950-NEXT: .LBB51_5:
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %rtn = atomicrmw xor ptr %gep1, i64 %data syncscope("agent") seq_cst
- %cast.rtn = bitcast i64 %rtn to <2 x float>
- ret <2 x float> %cast.rtn
-}
-
-define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i64_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB52_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB52_4
-; GFX1250-NEXT: .LBB52_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_endpgm
-; GFX1250-NEXT: .LBB52_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB52_2
-; GFX1250-NEXT: .LBB52_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX1250-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[0:1], off
-; GFX1250-NEXT: s_endpgm
-;
-; GFX950-LABEL: flat_xor_saddr_i64_nortn:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB52_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB52_4
-; GFX950-NEXT: .LBB52_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_endpgm
-; GFX950-NEXT: .LBB52_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB52_2
-; GFX950-NEXT: .LBB52_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX950-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off
-; GFX950-NEXT: s_endpgm
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %unused = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst
- ret void
-}
-
-define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i64_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4
-; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB53_3
-; GFX1250-NEXT: ; %bb.1: ; %Flow
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execnz .LBB53_4
-; GFX1250-NEXT: .LBB53_2: ; %atomicrmw.phi
-; GFX1250-NEXT: s_endpgm
-; GFX1250-NEXT: .LBB53_3: ; %atomicrmw.global
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1250-NEXT: ; implicit-def: $vgpr3
-; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB53_2
-; GFX1250-NEXT: .LBB53_4: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX1250-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX1250-NEXT: scratch_store_b64 v4, v[0:1], off
-; GFX1250-NEXT: s_endpgm
-;
-; GFX950-LABEL: flat_xor_saddr_i64_nortn_neg128:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
-; GFX950-NEXT: v_mov_b32_e32 v2, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
-; GFX950-NEXT: s_movk_i32 s2, 0xff80
-; GFX950-NEXT: s_mov_b32 s3, -1
-; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
-; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB53_3
-; GFX950-NEXT: ; %bb.1: ; %Flow
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execnz .LBB53_4
-; GFX950-NEXT: .LBB53_2: ; %atomicrmw.phi
-; GFX950-NEXT: s_endpgm
-; GFX950-NEXT: .LBB53_3: ; %atomicrmw.global
-; GFX950-NEXT: buffer_wbl2 sc1
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: buffer_inv sc1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX950-NEXT: s_cbranch_execz .LBB53_2
-; GFX950-NEXT: .LBB53_4: ; %atomicrmw.private
-; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX950-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off
-; GFX950-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 7f4ab3a39cb29..d0e3ce4f1a430 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -625,11 +625,31 @@ define amdgpu_ps <3 x half> @test_fmaximum_v3f16_ss(<3 x half> inreg %a, <3 x ha
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX1170-LABEL: test_fmaximum_v3f16_ss:
-; GFX1170: ; %bb.0:
-; GFX1170-NEXT: v_pk_maximum_f16 v0, s0, s2
-; GFX1170-NEXT: v_pk_maximum_f16 v1, s1, s3
-; GFX1170-NEXT: ; return to shader part epilog
+; GFX1170-SDAG-LABEL: test_fmaximum_v3f16_ss:
+; GFX1170-SDAG: ; %bb.0:
+; GFX1170-SDAG-NEXT: v_pk_maximum_f16 v0, s0, s2
+; GFX1170-SDAG-NEXT: v_pk_maximum_f16 v1, s1, s3
+; GFX1170-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1170-GISEL-TRUE16-LABEL: test_fmaximum_v3f16_ss:
+; GFX1170-GISEL-TRUE16: ; %bb.0:
+; GFX1170-GISEL-TRUE16-NEXT: v_maximum_f16 v0.l, s1, s3
+; GFX1170-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1170-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1170-GISEL-TRUE16-NEXT: v_pk_maximum_f16 v0, s0, s2
+; GFX1170-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1170-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1170-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1170-GISEL-FAKE16-LABEL: test_fmaximum_v3f16_ss:
+; GFX1170-GISEL-FAKE16: ; %bb.0:
+; GFX1170-GISEL-FAKE16-NEXT: v_maximum_f16 v1, s1, s3
+; GFX1170-GISEL-FAKE16-NEXT: v_pk_maximum_f16 v0, s0, s2
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1170-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: test_fmaximum_v3f16_ss:
; GFX12-SDAG: ; %bb.0:
@@ -1212,22 +1232,44 @@ define amdgpu_kernel void @fmaximumi_f32_move_to_valu(ptr addrspace(1) %out, ptr
}
define amdgpu_kernel void @fmaximum_f16_move_to_valu(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) {
-; GFX9-LABEL: fmaximum_f16_move_to_valu:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x7e00
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_ushort v2, v0, s[6:7] glc
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f16_e32 v4, v1, v2
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: global_store_short v0, v1, s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: fmaximum_f16_move_to_valu:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: global_load_ushort v1, v0, s[2:3] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_load_ushort v2, v0, s[6:7] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: v_max_f16_e32 v4, v1, v2
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: fmaximum_f16_move_to_valu:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ushort v1, v0, s[2:3] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ushort v2, v0, s[6:7] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: v_max_f16_e32 v2, s2, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s2, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s2, 0x7e00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; GFX1170-SDAG-TRUE16-LABEL: fmaximum_f16_move_to_valu:
; GFX1170-SDAG-TRUE16: ; %bb.0:
@@ -1285,7 +1327,10 @@ define amdgpu_kernel void @fmaximum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1170-GISEL-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] glc dlc
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1170-GISEL-FAKE16-NEXT: v_maximum_f16 v1, v1, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v2
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_maximum_f16 v1, s2, s3
; GFX1170-GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1170-GISEL-FAKE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index 7dfdae48a52f0..c7548cd331be0 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -1925,47 +1925,102 @@ define amdgpu_ps half @test_fmax_f16_v_ieee_off(half %a, half %b) #0 {
}
define amdgpu_kernel void @test_fmax_f16_s_ieee_on(ptr addrspace(1) %out, half inreg %a, half inreg %b) #0 {
-; GFX8-LABEL: test_fmax_f16_s_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s6, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX8-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX8-NEXT: v_max_f16_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmax_f16_s_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: test_fmax_f16_s_ieee_on:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX9-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX9-NEXT: v_max_f16_e32 v0, v1, v0
-; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX9-NEXT: s_endpgm
+; GFX8-GISEL-LABEL: test_fmax_f16_s_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-GISEL-NEXT: buffer_load_ushort v0, off, s[4:7], 0 offset:46
+; GFX8-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX8-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s1, s1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX8-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: test_fmax_f16_s_ieee_on:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-NEXT: v_max_num_f16_e64 v0.h, s2, s2
-; GFX12-NEXT: v_max_num_f16_e64 v0.l, s3, s3
-; GFX12-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-NEXT: s_mov_b32 s2, -1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f16_e32 v0.l, v0.h, v0.l
-; GFX12-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: test_fmax_f16_s_ieee_on:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: test_fmax_f16_s_ieee_on:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-GISEL-NEXT: buffer_load_ushort v0, off, s[4:7], 0 offset:46
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e64 v1, s2, s2
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-GISEL-NEXT: v_max_f16_e64 v0, s2, s2
+; GFX9-GISEL-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: test_fmax_f16_s_ieee_on:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s3, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f16_e32 v0.l, v0.h, v0.l
+; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: test_fmax_f16_s_ieee_on:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: s_load_u16 s2, s[4:5], 0x2c
+; GFX12-GISEL-NEXT: s_load_u16 s3, s[4:5], 0x2e
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v1.l, s3, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX12-GISEL-NEXT: s_max_num_f16 s2, s2, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-GISEL-NEXT: s_endpgm
%val = call half @llvm.maxnum.f16(half %a, half %b)
store half %val, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 10ac44995f1bb..4a1d8358ff754 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -625,11 +625,31 @@ define amdgpu_ps <3 x half> @test_fminimum_v3f16_ss(<3 x half> inreg %a, <3 x ha
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX1170-LABEL: test_fminimum_v3f16_ss:
-; GFX1170: ; %bb.0:
-; GFX1170-NEXT: v_pk_minimum_f16 v0, s0, s2
-; GFX1170-NEXT: v_pk_minimum_f16 v1, s1, s3
-; GFX1170-NEXT: ; return to shader part epilog
+; GFX1170-SDAG-LABEL: test_fminimum_v3f16_ss:
+; GFX1170-SDAG: ; %bb.0:
+; GFX1170-SDAG-NEXT: v_pk_minimum_f16 v0, s0, s2
+; GFX1170-SDAG-NEXT: v_pk_minimum_f16 v1, s1, s3
+; GFX1170-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1170-GISEL-TRUE16-LABEL: test_fminimum_v3f16_ss:
+; GFX1170-GISEL-TRUE16: ; %bb.0:
+; GFX1170-GISEL-TRUE16-NEXT: v_minimum_f16 v0.l, s1, s3
+; GFX1170-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1170-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1170-GISEL-TRUE16-NEXT: v_pk_minimum_f16 v0, s0, s2
+; GFX1170-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1170-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1170-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1170-GISEL-FAKE16-LABEL: test_fminimum_v3f16_ss:
+; GFX1170-GISEL-FAKE16: ; %bb.0:
+; GFX1170-GISEL-FAKE16-NEXT: v_minimum_f16 v1, s1, s3
+; GFX1170-GISEL-FAKE16-NEXT: v_pk_minimum_f16 v0, s0, s2
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1170-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: test_fminimum_v3f16_ss:
; GFX12-SDAG: ; %bb.0:
@@ -1212,22 +1232,44 @@ define amdgpu_kernel void @fminimumi_f32_move_to_valu(ptr addrspace(1) %out, ptr
}
define amdgpu_kernel void @fminimum_f16_move_to_valu(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) {
-; GFX9-LABEL: fminimum_f16_move_to_valu:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x7e00
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_ushort v2, v0, s[6:7] glc
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_min_f16_e32 v4, v1, v2
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: global_store_short v0, v1, s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: fminimum_f16_move_to_valu:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: global_load_ushort v1, v0, s[2:3] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_load_ushort v2, v0, s[6:7] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: v_min_f16_e32 v4, v1, v2
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: fminimum_f16_move_to_valu:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ushort v1, v0, s[2:3] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ushort v2, v0, s[6:7] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: v_min_f16_e32 v2, s2, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s2, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s2, 0x7e00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; GFX1170-SDAG-TRUE16-LABEL: fminimum_f16_move_to_valu:
; GFX1170-SDAG-TRUE16: ; %bb.0:
@@ -1285,7 +1327,10 @@ define amdgpu_kernel void @fminimum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1170-GISEL-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] glc dlc
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1170-GISEL-FAKE16-NEXT: v_minimum_f16 v1, v1, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v2
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_minimum_f16 v1, s2, s3
; GFX1170-GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1170-GISEL-FAKE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index d5bec944a7ba8..0ab8c19828666 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -1838,47 +1838,102 @@ define amdgpu_ps half @test_fmin_f16_v_ieee_off(half %a, half %b) #0 {
}
define amdgpu_kernel void @test_fmin_f16_s_ieee_on(ptr addrspace(1) %out, half inreg %a, half inreg %b) #0 {
-; GFX8-LABEL: test_fmin_f16_s_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s6, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX8-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX8-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmin_f16_s_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: test_fmin_f16_s_ieee_on:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX9-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX9-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX9-NEXT: s_endpgm
+; GFX8-GISEL-LABEL: test_fmin_f16_s_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-GISEL-NEXT: buffer_load_ushort v0, off, s[4:7], 0 offset:46
+; GFX8-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX8-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s1, s1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX8-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: test_fmin_f16_s_ieee_on:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-NEXT: v_max_num_f16_e64 v0.h, s2, s2
-; GFX12-NEXT: v_max_num_f16_e64 v0.l, s3, s3
-; GFX12-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-NEXT: s_mov_b32 s2, -1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f16_e32 v0.l, v0.h, v0.l
-; GFX12-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: test_fmin_f16_s_ieee_on:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX9-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: test_fmin_f16_s_ieee_on:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-GISEL-NEXT: buffer_load_ushort v0, off, s[4:7], 0 offset:46
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e64 v1, s2, s2
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-GISEL-NEXT: v_max_f16_e64 v0, s2, s2
+; GFX9-GISEL-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: test_fmin_f16_s_ieee_on:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s3, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f16_e32 v0.l, v0.h, v0.l
+; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: test_fmin_f16_s_ieee_on:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: s_load_u16 s2, s[4:5], 0x2c
+; GFX12-GISEL-NEXT: s_load_u16 s3, s[4:5], 0x2e
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v1.l, s3, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX12-GISEL-NEXT: s_min_num_f16 s2, s2, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-GISEL-NEXT: s_endpgm
%val = call half @llvm.minnum.f16(half %a, half %b)
store half %val, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index a21089e4e7485..60f41607375f2 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -53,15 +53,15 @@ define i32 @flat_offset_maybe_oob(ptr %p, i32 %i) {
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: flat_offset_maybe_oob:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 12
-; GFX942-NEXT: flat_load_dword v0, v[0:1]
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: flat_offset_maybe_oob:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 12
+; GFX942-SDAG-NEXT: flat_load_dword v0, v[0:1]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_offset_maybe_oob:
; GFX11: ; %bb.0:
@@ -100,6 +100,18 @@ define i32 @flat_offset_maybe_oob(ptr %p, i32 %i) {
; GFX12-NEXT: flat_load_b32 v0, v[0:1]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: flat_offset_maybe_oob:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 12, v0
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: flat_load_dword v0, v[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%idx = add nsw i32 %i, 3
%arrayidx = getelementptr inbounds i32, ptr %p, i32 %idx
%l = load i32, ptr %arrayidx
@@ -280,105 +292,193 @@ define i32 @flat_offset_inbounds(ptr %p, i32 %i) {
}
define void @flat_offset_inbounds_wide(ptr %p, ptr %pout, i32 %i) {
-; GFX90A-LABEL: flat_offset_inbounds_wide:
-; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX90A-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 28, v0
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: flat_load_dword v10, v[4:5]
-; GFX90A-NEXT: flat_load_dwordx4 v[6:9], v[0:1] offset:12
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_store_dword v[2:3], v10 offset:16
-; GFX90A-NEXT: flat_store_dwordx4 v[2:3], v[6:9]
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: s_setpc_b64 s[30:31]
+; GFX90A-SDAG-LABEL: flat_offset_inbounds_wide:
+; GFX90A-SDAG: ; %bb.0:
+; GFX90A-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX90A-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX90A-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX90A-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX90A-SDAG-NEXT: v_add_co_u32_e32 v4, vcc, 28, v0
+; GFX90A-SDAG-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX90A-SDAG-NEXT: flat_load_dword v10, v[4:5]
+; GFX90A-SDAG-NEXT: flat_load_dwordx4 v[6:9], v[0:1] offset:12
+; GFX90A-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-NEXT: flat_store_dword v[2:3], v10 offset:16
+; GFX90A-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[6:9]
+; GFX90A-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_offset_inbounds_wide:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, 28
-; GFX10-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: flat_load_dword v8, v[4:5]
-; GFX10-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
-; GFX10-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
-; GFX10-NEXT: flat_store_dword v[2:3], v8 offset:16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(1)
-; GFX10-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: flat_offset_inbounds_wide:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX10-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX10-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: flat_load_dword v8, v[4:5]
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX10-SDAG-NEXT: flat_store_dword v[2:3], v8 offset:16
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(1)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: flat_offset_inbounds_wide:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 28
-; GFX942-NEXT: flat_load_dword v10, v[4:5]
-; GFX942-NEXT: flat_load_dwordx4 v[6:9], v[0:1] offset:12
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_store_dword v[2:3], v10 offset:16
-; GFX942-NEXT: flat_store_dwordx4 v[2:3], v[6:9]
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: flat_offset_inbounds_wide:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 28
+; GFX942-SDAG-NEXT: flat_load_dword v10, v[4:5]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[6:9], v[0:1] offset:12
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_dword v[2:3], v10 offset:16
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[6:9]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_offset_inbounds_wide:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, v0, 28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: flat_load_b32 v8, v[4:5]
-; GFX11-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
-; GFX11-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
-; GFX11-NEXT: flat_store_b32 v[2:3], v8 offset:16
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(1)
-; GFX11-NEXT: flat_store_b128 v[2:3], v[4:7]
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: flat_offset_inbounds_wide:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX11-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, 28
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: flat_load_b32 v8, v[4:5]
+; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX11-SDAG-NEXT: flat_store_b32 v[2:3], v8 offset:16
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(1)
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[4:7]
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: flat_offset_inbounds_wide:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, v0, 28
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: flat_load_b32 v8, v[4:5]
-; GFX12-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x101
-; GFX12-NEXT: flat_store_b32 v[2:3], v8 offset:16
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x1
-; GFX12-NEXT: flat_store_b128 v[2:3], v[4:7]
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: flat_offset_inbounds_wide:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, 28
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: flat_load_b32 v8, v[4:5]
+; GFX12-SDAG-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x101
+; GFX12-SDAG-NEXT: flat_store_b32 v[2:3], v8 offset:16
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x1
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[4:7]
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-GISEL-LABEL: flat_offset_inbounds_wide:
+; GFX90A-GISEL: ; %bb.0:
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX90A-GISEL-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX90A-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX90A-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
+; GFX90A-GISEL-NEXT: flat_load_dword v8, v[0:1] offset:28
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX90A-GISEL-NEXT: flat_store_dword v[2:3], v8 offset:16
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: flat_offset_inbounds_wide:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX10-GISEL-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
+; GFX10-GISEL-NEXT: flat_load_dword v0, v[0:1] offset:28
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(1)
+; GFX10-GISEL-NEXT: flat_store_dword v[2:3], v0 offset:16
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: flat_offset_inbounds_wide:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
+; GFX942-GISEL-NEXT: flat_load_dword v8, v[0:1] offset:28
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX942-GISEL-NEXT: flat_store_dword v[2:3], v8 offset:16
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: flat_offset_inbounds_wide:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:28
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[4:7]
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(1)
+; GFX11-GISEL-NEXT: flat_store_b32 v[2:3], v0 offset:16
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: flat_offset_inbounds_wide:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:28
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x101
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[4:7]
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x1
+; GFX12-GISEL-NEXT: flat_store_b32 v[2:3], v0 offset:16
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
%p.1 = getelementptr inbounds i32, ptr %p, i32 %i
%arrayidx = getelementptr inbounds i32, ptr %p.1, i32 3
%l = load <5 x i32>, ptr %arrayidx
@@ -387,275 +487,454 @@ define void @flat_offset_inbounds_wide(ptr %p, ptr %pout, i32 %i) {
}
define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
-; GFX90A-MUBUF-LABEL: flat_offset_inbounds_very_wide:
-; GFX90A-MUBUF: ; %bb.0:
-; GFX90A-MUBUF-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-MUBUF-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX90A-MUBUF-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
-; GFX90A-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
-; GFX90A-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
-; GFX90A-MUBUF-NEXT: v_add_co_u32_e64 v6, s[4:5], 28, v0
-; GFX90A-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
-; GFX90A-MUBUF-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, v1, s[4:5]
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
-; GFX90A-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-MUBUF-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
-; GFX90A-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
-; GFX90A-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX90A-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
-; GFX90A-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
-; GFX90A-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
-; GFX90A-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
-; GFX90A-MUBUF-NEXT: flat_store_dword v[4:5], v50
-; GFX90A-MUBUF-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
-; GFX90A-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-MUBUF-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX90A-FLATSCR-LABEL: flat_offset_inbounds_very_wide:
-; GFX90A-FLATSCR: ; %bb.0:
-; GFX90A-FLATSCR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-FLATSCR-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX90A-FLATSCR-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
-; GFX90A-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
-; GFX90A-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
-; GFX90A-FLATSCR-NEXT: v_add_co_u32_e64 v6, s[0:1], 28, v0
-; GFX90A-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
-; GFX90A-FLATSCR-NEXT: v_addc_co_u32_e64 v7, s[0:1], 0, v1, s[0:1]
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
-; GFX90A-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-FLATSCR-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
-; GFX90A-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
-; GFX90A-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX90A-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
-; GFX90A-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
-; GFX90A-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
-; GFX90A-FLATSCR-NEXT: flat_store_dword v[4:5], v50
-; GFX90A-FLATSCR-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
-; GFX90A-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-FLATSCR-NEXT: s_setpc_b64 s[30:31]
+; GFX90A-SDAG-MUBUF-LABEL: flat_offset_inbounds_very_wide:
+; GFX90A-SDAG-MUBUF: ; %bb.0:
+; GFX90A-SDAG-MUBUF-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-MUBUF-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX90A-SDAG-MUBUF-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e64 v6, s[4:5], 28, v0
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
+; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, v1, s[4:5]
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
+; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
+; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
+; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
+; GFX90A-SDAG-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dword v[4:5], v50
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
+; GFX90A-SDAG-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-MUBUF-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_offset_inbounds_very_wide:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX10-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
-; GFX10-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX10-NEXT: s_clause 0x8
-; GFX10-NEXT: flat_load_dwordx4 v[4:7], v[36:37] offset:80
-; GFX10-NEXT: flat_load_dwordx4 v[8:11], v[36:37] offset:96
-; GFX10-NEXT: flat_load_dwordx4 v[12:15], v[36:37] offset:48
-; GFX10-NEXT: flat_load_dwordx4 v[16:19], v[36:37] offset:64
-; GFX10-NEXT: flat_load_dwordx4 v[20:23], v[36:37] offset:16
-; GFX10-NEXT: flat_load_dwordx4 v[24:27], v[36:37] offset:32
-; GFX10-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:12
-; GFX10-NEXT: flat_load_dwordx4 v[32:35], v[36:37]
-; GFX10-NEXT: flat_load_dwordx4 v[36:39], v[36:37] offset:112
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
-; GFX10-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX10-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
-; GFX10-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX10-NEXT: s_waitcnt vmcnt(8) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
-; GFX10-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[0:1], v[8:11] offset:64
-; GFX10-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX10-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
-; GFX10-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:32
-; GFX10-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
-; GFX10-NEXT: s_waitcnt vmcnt(2) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[2:3], v[28:31]
-; GFX10-NEXT: s_waitcnt vmcnt(1) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
-; GFX10-NEXT: flat_store_dword v[48:49], v38
-; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[36:37] offset:128
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX90A-SDAG-FLATSCR-LABEL: flat_offset_inbounds_very_wide:
+; GFX90A-SDAG-FLATSCR: ; %bb.0:
+; GFX90A-SDAG-FLATSCR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-FLATSCR-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX90A-SDAG-FLATSCR-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e64 v6, s[0:1], 28, v0
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
+; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e64 v7, s[0:1], 0, v1, s[0:1]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
+; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
+; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
+; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
+; GFX90A-SDAG-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dword v[4:5], v50
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
+; GFX90A-SDAG-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-FLATSCR-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: flat_offset_inbounds_very_wide:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
-; GFX942-NEXT: s_mov_b64 s[2:3], 0x5c
-; GFX942-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[2:3], 0x4c
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[2:3], 0x7c
-; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[2:3], 0x6c
-; GFX942-NEXT: v_lshl_add_u64 v[16:17], v[0:1], 0, s[2:3]
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 28
-; GFX942-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, 60
-; GFX942-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, 44
-; GFX942-NEXT: flat_load_dwordx4 v[18:21], v[16:17]
-; GFX942-NEXT: flat_load_dwordx4 v[22:25], v[12:13]
-; GFX942-NEXT: flat_load_dwordx4 v[26:29], v[14:15]
-; GFX942-NEXT: ; kill: killed $vgpr12_vgpr13
-; GFX942-NEXT: ; kill: killed $vgpr14_vgpr15
-; GFX942-NEXT: ; kill: killed $vgpr16_vgpr17
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: flat_load_dwordx4 v[12:15], v[8:9]
-; GFX942-NEXT: flat_load_dwordx4 v[30:33], v[10:11]
-; GFX942-NEXT: flat_load_dwordx4 v[34:37], v[4:5]
-; GFX942-NEXT: flat_load_dwordx4 v[48:51], v[6:7]
-; GFX942-NEXT: flat_load_dwordx4 v[52:55], v[0:1] offset:12
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x8c
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942-NEXT: flat_load_dwordx4 a[0:3], v[0:1]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x60
-; GFX942-NEXT: s_mov_b64 s[2:3], 0x70
-; GFX942-NEXT: s_mov_b64 s[4:5], 0x50
-; GFX942-NEXT: s_mov_b64 s[6:7], 0x88
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[2:3], 0, s[2:3]
-; GFX942-NEXT: v_lshl_add_u64 v[6:7], v[2:3], 0, s[4:5]
-; GFX942-NEXT: v_lshl_add_u64 v[8:9], v[2:3], 0, 48
-; GFX942-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, s[6:7]
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_store_dwordx4 v[0:1], v[18:21]
-; GFX942-NEXT: flat_store_dwordx4 v[4:5], v[26:29]
-; GFX942-NEXT: flat_store_dwordx4 v[2:3], v[22:25] offset:64
-; GFX942-NEXT: flat_store_dwordx4 v[6:7], v[30:33]
-; GFX942-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
-; GFX942-NEXT: flat_store_dwordx4 v[8:9], v[48:51]
-; GFX942-NEXT: flat_store_dwordx4 v[2:3], v[52:55]
-; GFX942-NEXT: flat_store_dwordx4 v[2:3], v[34:37] offset:16
-; GFX942-NEXT: flat_store_dword v[10:11], a2
-; GFX942-NEXT: flat_store_dwordx2 v[2:3], a[0:1] offset:128
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: flat_offset_inbounds_very_wide:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX10-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX10-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: s_clause 0x8
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[36:37] offset:80
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[36:37] offset:96
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[36:37] offset:48
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[36:37] offset:64
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[36:37] offset:16
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[36:37] offset:32
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:12
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[36:37]
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[36:39], v[36:37] offset:112
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(8) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11] offset:64
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:32
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(2) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[28:31]
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(1) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:16
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
+; GFX10-SDAG-NEXT: flat_store_dword v[48:49], v38
+; GFX10-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[36:37] offset:128
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_offset_inbounds_very_wide:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX11-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX11-NEXT: s_clause 0x7
-; GFX11-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX11-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX11-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX11-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX11-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX11-NEXT: flat_load_b128 v[24:27], v[36:37]
-; GFX11-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX11-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX11-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX11-NEXT: s_clause 0x7
-; GFX11-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX11-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX11-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
-; GFX11-NEXT: flat_store_b128 v[0:1], v[16:19]
-; GFX11-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
-; GFX11-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
-; GFX11-NEXT: flat_store_b128 v[2:3], v[28:31]
-; GFX11-NEXT: flat_store_b32 v[48:49], v34
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
-; GFX11-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: flat_offset_inbounds_very_wide:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x5c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x4c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x7c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x6c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[16:17], v[0:1], 0, s[2:3]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 28
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, 60
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, 44
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[18:21], v[16:17]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[22:25], v[12:13]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[26:29], v[14:15]
+; GFX942-SDAG-NEXT: ; kill: killed $vgpr12_vgpr13
+; GFX942-SDAG-NEXT: ; kill: killed $vgpr14_vgpr15
+; GFX942-SDAG-NEXT: ; kill: killed $vgpr16_vgpr17
+; GFX942-SDAG-NEXT: s_nop 0
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[8:9]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[30:33], v[10:11]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[34:37], v[4:5]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[48:51], v[6:7]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 v[52:55], v[0:1] offset:12
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x8c
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: flat_load_dwordx4 a[0:3], v[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x60
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x50
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x88
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[2:3], 0, s[2:3]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[2:3], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[2:3], 0, 48
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, s[6:7]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[18:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[26:29]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[22:25] offset:64
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[6:7], v[30:33]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[48:51]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[52:55]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[34:37] offset:16
+; GFX942-SDAG-NEXT: flat_store_dword v[10:11], a2
+; GFX942-SDAG-NEXT: flat_store_dwordx2 v[2:3], a[0:1] offset:128
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: flat_offset_inbounds_very_wide:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX12-NEXT: s_clause 0x7
-; GFX12-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX12-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX12-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX12-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX12-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX12-NEXT: flat_load_b128 v[24:27], v[36:37]
-; GFX12-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX12-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX12-NEXT: s_clause 0x7
-; GFX12-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX12-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX12-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
-; GFX12-NEXT: flat_store_b128 v[0:1], v[16:19]
-; GFX12-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
-; GFX12-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
-; GFX12-NEXT: flat_store_b128 v[2:3], v[28:31]
-; GFX12-NEXT: flat_store_b32 v[48:49], v34
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x8
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
-; GFX12-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: flat_offset_inbounds_very_wide:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX11-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT: s_clause 0x7
+; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
+; GFX11-SDAG-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
+; GFX11-SDAG-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
+; GFX11-SDAG-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
+; GFX11-SDAG-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
+; GFX11-SDAG-NEXT: flat_load_b128 v[24:27], v[36:37]
+; GFX11-SDAG-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
+; GFX11-SDAG-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
+; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX11-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
+; GFX11-SDAG-NEXT: s_clause 0x7
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[16:19]
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[28:31]
+; GFX11-SDAG-NEXT: flat_store_b32 v[48:49], v34
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
+; GFX11-SDAG-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: flat_offset_inbounds_very_wide:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
+; GFX12-SDAG-NEXT: s_clause 0x7
+; GFX12-SDAG-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
+; GFX12-SDAG-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
+; GFX12-SDAG-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
+; GFX12-SDAG-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
+; GFX12-SDAG-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
+; GFX12-SDAG-NEXT: flat_load_b128 v[24:27], v[36:37]
+; GFX12-SDAG-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
+; GFX12-SDAG-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX12-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
+; GFX12-SDAG-NEXT: s_clause 0x7
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[16:19]
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[28:31]
+; GFX12-SDAG-NEXT: flat_store_b32 v[48:49], v34
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x8
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
+; GFX12-SDAG-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-GISEL-LABEL: flat_offset_inbounds_very_wide:
+; GFX90A-GISEL: ; %bb.0:
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX90A-GISEL-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX90A-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX90A-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[8:11], v[0:1] offset:28
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[12:15], v[0:1] offset:44
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[16:19], v[0:1] offset:60
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[20:23], v[0:1] offset:76
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[24:27], v[0:1] offset:92
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:108
+; GFX90A-GISEL-NEXT: flat_load_dwordx4 v[32:35], v[0:1] offset:124
+; GFX90A-GISEL-NEXT: flat_load_dwordx3 v[36:38], v[0:1] offset:140
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[8:11] offset:16
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[16:19] offset:48
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:64
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[24:27] offset:80
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:96
+; GFX90A-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:112
+; GFX90A-GISEL-NEXT: flat_store_dwordx3 v[2:3], v[36:38] offset:128
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: flat_offset_inbounds_very_wide:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX10-GISEL-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX10-GISEL-NEXT: s_clause 0x8
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[8:11], v[0:1] offset:28
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[12:15], v[0:1] offset:44
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[16:19], v[0:1] offset:60
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[20:23], v[0:1] offset:76
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[24:27], v[0:1] offset:92
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:108
+; GFX10-GISEL-NEXT: flat_load_dwordx4 v[32:35], v[0:1] offset:124
+; GFX10-GISEL-NEXT: flat_load_dwordx3 v[36:38], v[0:1] offset:140
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(8) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[8:11] offset:16
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[16:19] offset:48
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:64
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[24:27] offset:80
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(2) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:96
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(1) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:112
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
+; GFX10-GISEL-NEXT: flat_store_dwordx3 v[2:3], v[36:38] offset:128
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: flat_offset_inbounds_very_wide:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[0:1] offset:12
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[8:11], v[0:1] offset:28
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[12:15], v[0:1] offset:44
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[16:19], v[0:1] offset:60
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[20:23], v[0:1] offset:76
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[24:27], v[0:1] offset:92
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:108
+; GFX942-GISEL-NEXT: flat_load_dwordx4 v[32:35], v[0:1] offset:124
+; GFX942-GISEL-NEXT: flat_load_dwordx3 v[36:38], v[0:1] offset:140
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[8:11] offset:16
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:32
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[16:19] offset:48
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:64
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[24:27] offset:80
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:96
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:112
+; GFX942-GISEL-NEXT: flat_store_dwordx3 v[2:3], v[36:38] offset:128
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: flat_offset_inbounds_very_wide:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
+; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX11-GISEL-NEXT: s_clause 0x8
+; GFX11-GISEL-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
+; GFX11-GISEL-NEXT: flat_load_b128 v[8:11], v[0:1] offset:28
+; GFX11-GISEL-NEXT: flat_load_b128 v[12:15], v[0:1] offset:44
+; GFX11-GISEL-NEXT: flat_load_b128 v[16:19], v[0:1] offset:60
+; GFX11-GISEL-NEXT: flat_load_b128 v[20:23], v[0:1] offset:76
+; GFX11-GISEL-NEXT: flat_load_b128 v[24:27], v[0:1] offset:92
+; GFX11-GISEL-NEXT: flat_load_b128 v[28:31], v[0:1] offset:108
+; GFX11-GISEL-NEXT: flat_load_b128 v[32:35], v[0:1] offset:124
+; GFX11-GISEL-NEXT: flat_load_b96 v[36:38], v[0:1] offset:140
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(8) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[4:7]
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[8:11] offset:16
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[12:15] offset:32
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[16:19] offset:48
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[20:23] offset:64
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[24:27] offset:80
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(2) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[28:31] offset:96
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(1) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b128 v[2:3], v[32:35] offset:112
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
+; GFX11-GISEL-NEXT: flat_store_b96 v[2:3], v[36:38] offset:128
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: flat_offset_inbounds_very_wide:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-GISEL-NEXT: s_clause 0x8
+; GFX12-GISEL-NEXT: flat_load_b128 v[4:7], v[0:1] offset:12
+; GFX12-GISEL-NEXT: flat_load_b128 v[8:11], v[0:1] offset:28
+; GFX12-GISEL-NEXT: flat_load_b128 v[12:15], v[0:1] offset:44
+; GFX12-GISEL-NEXT: flat_load_b128 v[16:19], v[0:1] offset:60
+; GFX12-GISEL-NEXT: flat_load_b128 v[20:23], v[0:1] offset:76
+; GFX12-GISEL-NEXT: flat_load_b128 v[24:27], v[0:1] offset:92
+; GFX12-GISEL-NEXT: flat_load_b128 v[28:31], v[0:1] offset:108
+; GFX12-GISEL-NEXT: flat_load_b128 v[32:35], v[0:1] offset:124
+; GFX12-GISEL-NEXT: flat_load_b96 v[36:38], v[0:1] offset:140
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x808
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[4:7]
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x708
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[8:11] offset:16
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x608
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[12:15] offset:32
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x508
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[16:19] offset:48
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x408
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[20:23] offset:64
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x308
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[24:27] offset:80
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x208
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[28:31] offset:96
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x108
+; GFX12-GISEL-NEXT: flat_store_b128 v[2:3], v[32:35] offset:112
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x8
+; GFX12-GISEL-NEXT: flat_store_b96 v[2:3], v[36:38] offset:128
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
%p.1 = getelementptr inbounds i32, ptr %p, i32 %i
%arrayidx = getelementptr inbounds i32, ptr %p.1, i32 3
%l = load <35 x i32>, ptr %arrayidx
@@ -663,16 +942,8 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10-GISEL: {{.*}}
; GFX10-GISEL-FLATSCR: {{.*}}
; GFX10-MUBUF: {{.*}}
-; GFX10-SDAG: {{.*}}
; GFX10-SDAG-FLATSCR: {{.*}}
-; GFX11-GISEL: {{.*}}
-; GFX11-SDAG: {{.*}}
-; GFX90A-GISEL: {{.*}}
; GFX90A-GISEL-FLATSCR: {{.*}}
-; GFX90A-SDAG: {{.*}}
-; GFX90A-SDAG-FLATSCR: {{.*}}
-; GFX942-GISEL: {{.*}}
-; GFX942-SDAG: {{.*}}
+; GFX90A-MUBUF: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 42622d98b7e14..1e86e628defb1 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -95,60 +95,120 @@ define i128 @fptosi_f64_to_i128(double %x) {
}
define i128 @fptoui_f64_to_i128(double %x) {
-; GCN-LABEL: fptoui_f64_to_i128:
-; GCN: ; %bb.0: ; %fp-to-i-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_bfe_u32 v6, v1, 20, 11
-; GCN-NEXT: v_mov_b32_e32 v7, 0
-; GCN-NEXT: s_mov_b64 s[4:5], 0x3fe
-; GCN-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
-; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: v_mov_b32_e32 v5, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB1_6
-; GCN-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
-; GCN-NEXT: s_mov_b64 s[4:5], 0x432
-; GCN-NEXT: v_and_b32_e32 v1, 0xfffff, v1
-; GCN-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
-; GCN-NEXT: v_or_b32_e32 v1, 0x100000, v1
-; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB1_3
-; GCN-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
-; GCN-NEXT: v_sub_u32_e32 v2, 0x473, v6
-; GCN-NEXT: v_add_u32_e32 v4, 0xfffffb8d, v6
-; GCN-NEXT: v_add_u32_e32 v7, 0xfffffbcd, v6
-; GCN-NEXT: v_lshrrev_b64 v[2:3], v2, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[4:5], v4, v[0:1]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7
-; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr6_vgpr7
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: .LBB1_3: ; %Flow
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GCN-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
-; GCN-NEXT: v_sub_u32_e32 v2, 0x433, v6
-; GCN-NEXT: v_lshrrev_b64 v[4:5], v2, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: ; %bb.5: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB1_6: ; %fp-to-i-cleanup
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: v_mov_b32_e32 v1, v5
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fptoui_f64_to_i128:
+; SDAG: ; %bb.0: ; %fp-to-i-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_bfe_u32 v6, v1, 20, 11
+; SDAG-NEXT: v_mov_b32_e32 v7, 0
+; SDAG-NEXT: s_mov_b64 s[4:5], 0x3fe
+; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v5, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB1_6
+; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; SDAG-NEXT: s_mov_b64 s[4:5], 0x432
+; SDAG-NEXT: v_and_b32_e32 v1, 0xfffff, v1
+; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
+; SDAG-NEXT: v_or_b32_e32 v1, 0x100000, v1
+; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB1_3
+; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; SDAG-NEXT: v_sub_u32_e32 v2, 0x473, v6
+; SDAG-NEXT: v_add_u32_e32 v4, 0xfffffb8d, v6
+; SDAG-NEXT: v_add_u32_e32 v7, 0xfffffbcd, v6
+; SDAG-NEXT: v_lshrrev_b64 v[2:3], v2, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[4:5], v4, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SDAG-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: .LBB1_3: ; %Flow
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; SDAG-NEXT: v_sub_u32_e32 v2, 0x433, v6
+; SDAG-NEXT: v_lshrrev_b64 v[4:5], v2, v[0:1]
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SDAG-NEXT: ; %bb.5: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB1_6: ; %fp-to-i-cleanup
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v0, v4
+; SDAG-NEXT: v_mov_b32_e32 v1, v5
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fptoui_f64_to_i128:
+; GISEL: ; %bb.0: ; %fp-to-i-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GISEL-NEXT: v_mov_b32_e32 v5, v1
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], 52, v[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v7, 0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0x3ff
+; GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-NEXT: v_bfe_u32 v6, v0, 0, 11
+; GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[6:7], v[1:2]
+; GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-NEXT: v_mov_b32_e32 v3, s7
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB1_6
+; GISEL-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x433
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: v_mov_b32_e32 v2, 0xfffff
+; GISEL-NEXT: v_mov_b32_e32 v3, 0x100000
+; GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[6:7], v[0:1]
+; GISEL-NEXT: v_and_or_b32 v5, v5, v2, v3
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
+; GISEL-NEXT: s_cbranch_execz .LBB1_3
+; GISEL-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; GISEL-NEXT: v_add_u32_e32 v7, 0xfffffbcd, v6
+; GISEL-NEXT: v_add_u32_e32 v6, 0xfffffb8d, v6
+; GISEL-NEXT: v_sub_u32_e32 v2, 64, v7
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v7, v[4:5]
+; GISEL-NEXT: v_lshrrev_b64 v[2:3], v2, v[4:5]
+; GISEL-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
+; GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr6
+; GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GISEL-NEXT: .LBB1_3: ; %Flow
+; GISEL-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GISEL-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, 0x433, v6
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], v0, v[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: ; %bb.5: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: .LBB1_6: ; %fp-to-i-cleanup
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%cvt = fptoui double %x to i128
ret i128 %cvt
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
index d11bc76649043..70d702b815b02 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
@@ -208,36 +208,67 @@ define i32 @test_signed_i32_f32(float %f) nounwind {
}
define i64 @test_signed_i64_f32(float %f) nounwind {
-; GFX7-LABEL: test_signed_i64_f32:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_trunc_f32_e32 v1, v0
-; GFX7-NEXT: s_mov_b32 s4, 0x2f800000
-; GFX7-NEXT: v_mul_f32_e64 v2, |v1|, s4
-; GFX7-NEXT: v_floor_f32_e32 v2, v2
-; GFX7-NEXT: s_mov_b32 s4, 0xcf800000
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX7-NEXT: v_fma_f32 v2, v2, s4, |v1|
-; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX7-NEXT: v_ashrrev_i32_e32 v1, 31, v1
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v1
-; GFX7-NEXT: s_mov_b32 s4, 0xdf000000
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v1
-; GFX7-NEXT: v_sub_i32_e32 v2, vcc, v2, v1
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: v_cmp_nle_f32_e32 vcc, s4, v0
-; GFX7-NEXT: s_mov_b32 s4, 0x5effffff
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc
-; GFX7-NEXT: v_cmp_lt_f32_e64 s[4:5], s4, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, -1, s[4:5]
-; GFX7-NEXT: v_cmp_u_f32_e64 s[6:7], v0, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[6:7]
-; GFX7-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX7-NEXT: v_bfrev_b32_e32 v2, -2
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, 0, s[6:7]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_signed_i64_f32:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0x2f800000
+; GFX7-ISEL-NEXT: v_mul_f32_e64 v2, |v1|, s4
+; GFX7-ISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0xcf800000
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v3, v2
+; GFX7-ISEL-NEXT: v_fma_f32 v2, v2, s4, |v1|
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; GFX7-ISEL-NEXT: v_xor_b32_e32 v3, v3, v1
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0xdf000000
+; GFX7-ISEL-NEXT: v_xor_b32_e32 v2, v2, v1
+; GFX7-ISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v1
+; GFX7-ISEL-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
+; GFX7-ISEL-NEXT: v_cmp_nle_f32_e32 vcc, s4, v0
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0x5effffff
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX7-ISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], s4, v0
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, -1, s[4:5]
+; GFX7-ISEL-NEXT: v_cmp_u_f32_e64 s[6:7], v0, v0
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[6:7]
+; GFX7-ISEL-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX7-ISEL-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7-ISEL-NEXT: v_bfrev_b32_e32 v2, -2
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, s[6:7]
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_signed_i64_f32:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_trunc_f32_e32 v1, v0
+; GFX7-GI-NEXT: v_mov_b32_e32 v2, 0x2f800000
+; GFX7-GI-NEXT: v_mul_f32_e64 v2, |v1|, v2
+; GFX7-GI-NEXT: v_floor_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0xcf800000
+; GFX7-GI-NEXT: v_fma_f32 v1, v2, v3, |v1|
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX7-GI-NEXT: v_bfrev_b32_e32 v4, 1
+; GFX7-GI-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GI-NEXT: v_xor_b32_e32 v2, v2, v3
+; GFX7-GI-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
+; GFX7-GI-NEXT: v_subb_u32_e32 v2, vcc, v2, v3, vcc
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0xdf000000
+; GFX7-GI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v3
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0x5effffff
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX7-GI-NEXT: v_bfrev_b32_e32 v4, -2
+; GFX7-GI-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX7-GI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_signed_i64_f32:
; GFX9: ; %bb.0:
@@ -295,38 +326,71 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc_lo
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_signed_i64_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_trunc_f32_e32 v1, v0
-; GFX12-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
-; GFX12-NEXT: v_floor_f32_e32 v2, v2
-; GFX12-NEXT: v_fma_f32 v3, 0xcf800000, v2, |v1|
-; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v1
-; GFX12-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX12-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v1
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v1
-; GFX12-NEXT: v_sub_co_u32 v3, vcc_lo, v3, v1
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_sub_co_ci_u32_e64 v1, null, v2, v1, vcc_lo
-; GFX12-NEXT: v_cmp_nle_f32_e32 vcc_lo, 0xdf000000, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v3, 0, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, 0x80000000, vcc_lo
-; GFX12-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x5effffff, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, -1, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, 0x7fffffff, vcc_lo
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc_lo
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_signed_i64_f32:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX12-ISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX12-ISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX12-ISEL-NEXT: v_fma_f32 v3, 0xcf800000, v2, |v1|
+; GFX12-ISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; GFX12-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX12-ISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX12-ISEL-NEXT: v_xor_b32_e32 v2, v2, v1
+; GFX12-ISEL-NEXT: v_xor_b32_e32 v3, v3, v1
+; GFX12-ISEL-NEXT: v_sub_co_u32 v3, vcc_lo, v3, v1
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v2, v1, vcc_lo
+; GFX12-ISEL-NEXT: v_cmp_nle_f32_e32 vcc_lo, 0xdf000000, v0
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v3, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0x80000000, vcc_lo
+; GFX12-ISEL-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x5effffff, v0
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, -1, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0x7fffffff, vcc_lo
+; GFX12-ISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc_lo
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_signed_i64_f32:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_trunc_f32_e32 v1, v0
+; GFX12-GI-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX12-GI-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX12-GI-NEXT: v_floor_f32_e32 v2, v2
+; GFX12-GI-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX12-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX12-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX12-GI-NEXT: v_xor_b32_e32 v2, v2, v3
+; GFX12-GI-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX12-GI-NEXT: v_sub_co_u32 v1, vcc_lo, v1, v3
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_sub_co_ci_u32_e64 v2, null, v2, v3, vcc_lo
+; GFX12-GI-NEXT: v_cmp_nle_f32_e32 vcc_lo, 0xdf000000, v0
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v2, v2, 0x80000000, vcc_lo
+; GFX12-GI-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x5effffff, v0
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v1, -1, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v2, v2, 0x7fffffff, vcc_lo
+; GFX12-GI-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v2, 0, vcc_lo
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i64 @llvm.fptosi.sat.i64.f32(float %f)
ret i64 %x
}
@@ -464,14 +528,23 @@ define i8 @test_s_signed_i8_f32(float inreg %f) nounwind {
}
define i16 @test_s_signed_i16_f32(float inreg %f) nounwind {
-; GFX7-LABEL: test_s_signed_i16_f32:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, s16
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_signed_i16_f32:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, s16
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v1
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_signed_i16_f32:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, s16
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, 0x8000
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v1, s4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_signed_i16_f32:
; GFX9: ; %bb.0:
@@ -987,34 +1060,64 @@ define i32 @test_signed_i32_f64(double %f) nounwind {
}
define i64 @test_signed_i64_f64(double %f) nounwind {
-; GFX7-LABEL: test_signed_i64_f64:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX7-NEXT: s_movk_i32 s4, 0xffe0
-; GFX7-NEXT: s_mov_b32 s8, 0
-; GFX7-NEXT: s_mov_b32 s9, 0xc1f00000
-; GFX7-NEXT: s_mov_b32 s5, 0xc3e00000
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: s_mov_b32 s7, 0x43dfffff
-; GFX7-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX7-NEXT: v_ldexp_f64 v[4:5], v[2:3], s4
-; GFX7-NEXT: s_mov_b32 s4, 0
-; GFX7-NEXT: v_cmp_nle_f64_e32 vcc, s[4:5], v[0:1]
-; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], s[6:7], v[0:1]
-; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[0:1], v[0:1]
-; GFX7-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
-; GFX7-NEXT: v_fma_f64 v[2:3], v[4:5], s[8:9], v[2:3]
-; GFX7-NEXT: v_cvt_i32_f64_e32 v7, v[4:5]
-; GFX7-NEXT: v_bfrev_b32_e32 v4, -2
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v7, v6, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v4, v5, v4, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v4, 0, s[6:7]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v0, v[2:3]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[6:7]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_signed_i64_f64:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0xffe0
+; GFX7-ISEL-NEXT: s_mov_b32 s8, 0
+; GFX7-ISEL-NEXT: s_mov_b32 s9, 0xc1f00000
+; GFX7-ISEL-NEXT: s_mov_b32 s5, 0xc3e00000
+; GFX7-ISEL-NEXT: s_mov_b32 s6, -1
+; GFX7-ISEL-NEXT: s_mov_b32 s7, 0x43dfffff
+; GFX7-ISEL-NEXT: v_bfrev_b32_e32 v6, 1
+; GFX7-ISEL-NEXT: v_ldexp_f64 v[4:5], v[2:3], s4
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0
+; GFX7-ISEL-NEXT: v_cmp_nle_f64_e32 vcc, s[4:5], v[0:1]
+; GFX7-ISEL-NEXT: v_cmp_lt_f64_e64 s[4:5], s[6:7], v[0:1]
+; GFX7-ISEL-NEXT: v_cmp_u_f64_e64 s[6:7], v[0:1], v[0:1]
+; GFX7-ISEL-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX7-ISEL-NEXT: v_fma_f64 v[2:3], v[4:5], s[8:9], v[2:3]
+; GFX7-ISEL-NEXT: v_cvt_i32_f64_e32 v7, v[4:5]
+; GFX7-ISEL-NEXT: v_bfrev_b32_e32 v4, -2
+; GFX7-ISEL-NEXT: v_cndmask_b32_e32 v5, v7, v6, vcc
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v4, v5, v4, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v1, v4, 0, s[6:7]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v0, v[2:3]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[6:7]
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_signed_i64_f64:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX7-GI-NEXT: v_mov_b32_e32 v4, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v5, 0x3df00000
+; GFX7-GI-NEXT: v_mov_b32_e32 v6, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v7, 0xc1f00000
+; GFX7-GI-NEXT: v_cmp_u_f64_e64 s[6:7], v[0:1], v[0:1]
+; GFX7-GI-NEXT: v_mul_f64 v[4:5], v[2:3], v[4:5]
+; GFX7-GI-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX7-GI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], v[2:3]
+; GFX7-GI-NEXT: v_mov_b32_e32 v6, -1
+; GFX7-GI-NEXT: v_mov_b32_e32 v7, 0x43dfffff
+; GFX7-GI-NEXT: v_cmp_gt_f64_e64 s[4:5], v[0:1], v[6:7]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v8, v[2:3]
+; GFX7-GI-NEXT: v_mov_b32_e32 v2, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0xc3e00000
+; GFX7-GI-NEXT: v_cmp_nge_f64_e32 vcc, v[0:1], v[2:3]
+; GFX7-GI-NEXT: v_cvt_i32_f64_e32 v1, v[4:5]
+; GFX7-GI-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX7-GI-NEXT: v_bfrev_b32_e32 v4, -2
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v2, v2, -1, s[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, v4, s[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[6:7]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, s[6:7]
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_signed_i64_f64:
; GFX9: ; %bb.0:
@@ -1067,34 +1170,61 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, 0, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_signed_i64_f64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX12-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0xc3e00000, v[0:1]
-; GFX12-NEXT: s_mov_b32 s0, -1
-; GFX12-NEXT: s_mov_b32 s1, 0x43dfffff
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmp_lt_f64_e64 s0, s[0:1], v[0:1]
-; GFX12-NEXT: v_cmp_u_f64_e64 s1, v[0:1], v[0:1]
-; GFX12-NEXT: v_ldexp_f64 v[4:5], v[2:3], 0xffffffe0
-; GFX12-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
-; GFX12-NEXT: v_fma_f64 v[2:3], 0xc1f00000, v[4:5], v[2:3]
-; GFX12-NEXT: v_cvt_i32_f64_e32 v4, v[4:5]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v2, v[2:3]
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v3, v4, 0x80000000, vcc_lo
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v3, v3, 0x7fffffff, s0
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v3, 0, s1
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, -1, s0
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, 0, s1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_signed_i64_f64:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX12-ISEL-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0xc3e00000, v[0:1]
+; GFX12-ISEL-NEXT: s_mov_b32 s0, -1
+; GFX12-ISEL-NEXT: s_mov_b32 s1, 0x43dfffff
+; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-ISEL-NEXT: v_cmp_lt_f64_e64 s0, s[0:1], v[0:1]
+; GFX12-ISEL-NEXT: v_cmp_u_f64_e64 s1, v[0:1], v[0:1]
+; GFX12-ISEL-NEXT: v_ldexp_f64 v[4:5], v[2:3], 0xffffffe0
+; GFX12-ISEL-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX12-ISEL-NEXT: v_fma_f64 v[2:3], 0xc1f00000, v[4:5], v[2:3]
+; GFX12-ISEL-NEXT: v_cvt_i32_f64_e32 v4, v[4:5]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v2, v[2:3]
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v3, v4, 0x80000000, vcc_lo
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0x7fffffff, s0
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0, s1
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, s0
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, 0, s1
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_signed_i64_f64:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX12-GI-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0xc3e00000, v[0:1]
+; GFX12-GI-NEXT: v_cmp_u_f64_e64 s1, v[0:1], v[0:1]
+; GFX12-GI-NEXT: v_mul_f64_e32 v[4:5], 0x3df00000, v[2:3]
+; GFX12-GI-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX12-GI-NEXT: v_fma_f64 v[2:3], 0xc1f00000, v[4:5], v[2:3]
+; GFX12-GI-NEXT: v_cvt_i32_f64_e32 v4, v[4:5]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v6, v[2:3]
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, -1 :: v_dual_mov_b32 v3, 0x43dfffff
+; GFX12-GI-NEXT: v_cmp_gt_f64_e64 s0, v[0:1], v[2:3]
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v4, 0x80000000, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v5, v6, 0, vcc_lo
+; GFX12-GI-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v2, v5, -1, s0
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v0, 0x7fffffff, s0
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v2, 0, s1
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, s1
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i64 @llvm.fptosi.sat.i64.f64(double %f)
ret i64 %x
}
@@ -2073,15 +2203,25 @@ define i8 @test_s_signed_i8_f16(half inreg %f) nounwind {
}
define i16 @test_s_signed_i16_f16(half inreg %f) nounwind {
-; GFX7-LABEL: test_s_signed_i16_f16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, s16
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_signed_i16_f16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, s16
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v1
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_signed_i16_f16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, s16
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, 0x8000
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v1, s4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_signed_i16_f16:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index c3bcb76434dbb..83c2773b9eb91 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -1614,49 +1614,94 @@ define <2 x i32> @test_signed_v2f64_v2i32_duplicate(<2 x double> %f) {
}
define <2 x i64> @test_signed_v2f64_v2i64(<2 x double> %f) {
-; GFX7-LABEL: test_signed_v2f64_v2i64:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
-; GFX7-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
-; GFX7-NEXT: s_movk_i32 s4, 0xffe0
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xc1f00000
-; GFX7-NEXT: s_mov_b32 s5, 0xc3e00000
-; GFX7-NEXT: s_mov_b32 s8, -1
-; GFX7-NEXT: s_mov_b32 s9, 0x43dfffff
-; GFX7-NEXT: v_ldexp_f64 v[8:9], v[4:5], s4
-; GFX7-NEXT: v_ldexp_f64 v[10:11], v[6:7], s4
-; GFX7-NEXT: s_mov_b32 s4, 0
-; GFX7-NEXT: v_cmp_nle_f64_e32 vcc, s[4:5], v[0:1]
-; GFX7-NEXT: v_cmp_nle_f64_e64 s[4:5], s[4:5], v[2:3]
-; GFX7-NEXT: v_cmp_u_f64_e64 s[10:11], v[0:1], v[0:1]
-; GFX7-NEXT: v_cmp_u_f64_e64 s[12:13], v[2:3], v[2:3]
-; GFX7-NEXT: v_bfrev_b32_e32 v12, 1
-; GFX7-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
-; GFX7-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
-; GFX7-NEXT: v_bfrev_b32_e32 v13, -2
-; GFX7-NEXT: v_fma_f64 v[4:5], v[8:9], s[6:7], v[4:5]
-; GFX7-NEXT: v_fma_f64 v[6:7], v[10:11], s[6:7], v[6:7]
-; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], s[8:9], v[0:1]
-; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], s[8:9], v[2:3]
-; GFX7-NEXT: v_cvt_i32_f64_e32 v14, v[8:9]
-; GFX7-NEXT: v_cvt_i32_f64_e32 v15, v[10:11]
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
-; GFX7-NEXT: v_cvt_u32_f64_e32 v0, v[4:5]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v2, v[6:7]
-; GFX7-NEXT: v_cndmask_b32_e64 v9, v15, v12, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, v13, s[6:7]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v13, s[8:9]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[6:7]
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, -1, s[8:9]
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v8, 0, s[10:11]
-; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, 0, s[12:13]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[10:11]
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[12:13]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_signed_v2f64_v2i64:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX7-ISEL-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0xffe0
+; GFX7-ISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-ISEL-NEXT: s_mov_b32 s7, 0xc1f00000
+; GFX7-ISEL-NEXT: s_mov_b32 s5, 0xc3e00000
+; GFX7-ISEL-NEXT: s_mov_b32 s8, -1
+; GFX7-ISEL-NEXT: s_mov_b32 s9, 0x43dfffff
+; GFX7-ISEL-NEXT: v_ldexp_f64 v[8:9], v[4:5], s4
+; GFX7-ISEL-NEXT: v_ldexp_f64 v[10:11], v[6:7], s4
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0
+; GFX7-ISEL-NEXT: v_cmp_nle_f64_e32 vcc, s[4:5], v[0:1]
+; GFX7-ISEL-NEXT: v_cmp_nle_f64_e64 s[4:5], s[4:5], v[2:3]
+; GFX7-ISEL-NEXT: v_cmp_u_f64_e64 s[10:11], v[0:1], v[0:1]
+; GFX7-ISEL-NEXT: v_cmp_u_f64_e64 s[12:13], v[2:3], v[2:3]
+; GFX7-ISEL-NEXT: v_bfrev_b32_e32 v12, 1
+; GFX7-ISEL-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX7-ISEL-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX7-ISEL-NEXT: v_bfrev_b32_e32 v13, -2
+; GFX7-ISEL-NEXT: v_fma_f64 v[4:5], v[8:9], s[6:7], v[4:5]
+; GFX7-ISEL-NEXT: v_fma_f64 v[6:7], v[10:11], s[6:7], v[6:7]
+; GFX7-ISEL-NEXT: v_cmp_lt_f64_e64 s[6:7], s[8:9], v[0:1]
+; GFX7-ISEL-NEXT: v_cmp_lt_f64_e64 s[8:9], s[8:9], v[2:3]
+; GFX7-ISEL-NEXT: v_cvt_i32_f64_e32 v14, v[8:9]
+; GFX7-ISEL-NEXT: v_cvt_i32_f64_e32 v15, v[10:11]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v0, v[4:5]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v2, v[6:7]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v9, v15, v12, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v8, v8, v13, s[6:7]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v9, v9, v13, s[8:9]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[6:7]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, -1, s[8:9]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v1, v8, 0, s[10:11]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v3, v9, 0, s[12:13]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[10:11]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[12:13]
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_signed_v2f64_v2i64:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX7-GI-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX7-GI-NEXT: v_mov_b32_e32 v8, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v9, 0x3df00000
+; GFX7-GI-NEXT: v_mov_b32_e32 v12, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v13, 0xc1f00000
+; GFX7-GI-NEXT: v_mov_b32_e32 v14, -1
+; GFX7-GI-NEXT: v_mov_b32_e32 v15, 0x43dfffff
+; GFX7-GI-NEXT: v_mul_f64 v[10:11], v[4:5], v[8:9]
+; GFX7-GI-NEXT: v_mul_f64 v[8:9], v[6:7], v[8:9]
+; GFX7-GI-NEXT: v_cmp_gt_f64_e64 s[10:11], v[2:3], v[14:15]
+; GFX7-GI-NEXT: v_cmp_u_f64_e64 s[12:13], v[2:3], v[2:3]
+; GFX7-GI-NEXT: v_cmp_gt_f64_e64 s[6:7], v[0:1], v[14:15]
+; GFX7-GI-NEXT: v_cmp_u_f64_e64 s[8:9], v[0:1], v[0:1]
+; GFX7-GI-NEXT: v_bfrev_b32_e32 v16, 1
+; GFX7-GI-NEXT: v_bfrev_b32_e32 v17, -2
+; GFX7-GI-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX7-GI-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX7-GI-NEXT: v_fma_f64 v[4:5], v[10:11], v[12:13], v[4:5]
+; GFX7-GI-NEXT: v_fma_f64 v[6:7], v[8:9], v[12:13], v[6:7]
+; GFX7-GI-NEXT: v_mov_b32_e32 v12, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v13, 0xc3e00000
+; GFX7-GI-NEXT: v_cmp_nge_f64_e64 s[4:5], v[2:3], v[12:13]
+; GFX7-GI-NEXT: v_cmp_nge_f64_e32 vcc, v[0:1], v[12:13]
+; GFX7-GI-NEXT: v_cvt_i32_f64_e32 v10, v[10:11]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v6, v[6:7]
+; GFX7-GI-NEXT: v_cvt_i32_f64_e32 v5, v[8:9]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v6, v6, 0, s[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v6, -1, s[10:11]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v2, v1, 0, s[12:13]
+; GFX7-GI-NEXT: v_cndmask_b32_e32 v1, v10, v16, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v3, v5, v16, s[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v4, v4, -1, s[6:7]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, v17, s[6:7]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v3, v3, v17, s[10:11]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[8:9]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, s[8:9]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v3, v3, 0, s[12:13]
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_signed_v2f64_v2i64:
; GFX9: ; %bb.0:
@@ -1739,49 +1784,91 @@ define <2 x i64> @test_signed_v2f64_v2i64(<2 x double> %f) {
; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, 0, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_signed_v2f64_v2i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
-; GFX12-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
-; GFX12-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0xc3e00000, v[0:1]
-; GFX12-NEXT: v_cmp_nle_f64_e64 s0, 0xc3e00000, v[2:3]
-; GFX12-NEXT: s_mov_b32 s2, -1
-; GFX12-NEXT: s_mov_b32 s3, 0x43dfffff
-; GFX12-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[2:3]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmp_lt_f64_e64 s1, s[2:3], v[0:1]
-; GFX12-NEXT: v_cmp_lt_f64_e64 s2, s[2:3], v[2:3]
-; GFX12-NEXT: v_cmp_u_f64_e64 s3, v[0:1], v[0:1]
-; GFX12-NEXT: v_ldexp_f64 v[8:9], v[4:5], 0xffffffe0
-; GFX12-NEXT: v_ldexp_f64 v[10:11], v[6:7], 0xffffffe0
-; GFX12-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
-; GFX12-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
-; GFX12-NEXT: v_fma_f64 v[4:5], 0xc1f00000, v[8:9], v[4:5]
-; GFX12-NEXT: v_fma_f64 v[6:7], 0xc1f00000, v[10:11], v[6:7]
-; GFX12-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
-; GFX12-NEXT: v_cvt_i32_f64_e32 v9, v[10:11]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v5, v[6:7]
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v6, v8, 0x80000000, vcc_lo
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v7, v9, 0x80000000, s0
-; GFX12-NEXT: v_cndmask_b32_e64 v6, v6, 0x7fffffff, s1
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v7, 0x7fffffff, s2
-; GFX12-NEXT: v_cndmask_b32_e64 v3, v0, 0, s4
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v4, 0, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v5, 0, s0
-; GFX12-NEXT: v_cndmask_b32_e64 v4, v1, -1, s1
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, -1, s2
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v6, 0, s3
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v4, 0, s3
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, 0, s4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_signed_v2f64_v2i64:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX12-ISEL-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX12-ISEL-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0xc3e00000, v[0:1]
+; GFX12-ISEL-NEXT: v_cmp_nle_f64_e64 s0, 0xc3e00000, v[2:3]
+; GFX12-ISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-ISEL-NEXT: s_mov_b32 s3, 0x43dfffff
+; GFX12-ISEL-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[2:3]
+; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-ISEL-NEXT: v_cmp_lt_f64_e64 s1, s[2:3], v[0:1]
+; GFX12-ISEL-NEXT: v_cmp_lt_f64_e64 s2, s[2:3], v[2:3]
+; GFX12-ISEL-NEXT: v_cmp_u_f64_e64 s3, v[0:1], v[0:1]
+; GFX12-ISEL-NEXT: v_ldexp_f64 v[8:9], v[4:5], 0xffffffe0
+; GFX12-ISEL-NEXT: v_ldexp_f64 v[10:11], v[6:7], 0xffffffe0
+; GFX12-ISEL-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX12-ISEL-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX12-ISEL-NEXT: v_fma_f64 v[4:5], 0xc1f00000, v[8:9], v[4:5]
+; GFX12-ISEL-NEXT: v_fma_f64 v[6:7], 0xc1f00000, v[10:11], v[6:7]
+; GFX12-ISEL-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
+; GFX12-ISEL-NEXT: v_cvt_i32_f64_e32 v9, v[10:11]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v5, v[6:7]
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v6, v8, 0x80000000, vcc_lo
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v7, v9, 0x80000000, s0
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v6, v6, 0x7fffffff, s1
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v7, 0x7fffffff, s2
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v3, v0, 0, s4
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v4, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v5, 0, s0
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v4, v1, -1, s1
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, -1, s2
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v6, 0, s3
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v4, 0, s3
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, s4
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_signed_v2f64_v2i64:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX12-GI-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX12-GI-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0xc3e00000, v[0:1]
+; GFX12-GI-NEXT: v_cmp_nle_f64_e64 s0, 0xc3e00000, v[2:3]
+; GFX12-GI-NEXT: v_cmp_u_f64_e64 s3, v[0:1], v[0:1]
+; GFX12-GI-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[2:3]
+; GFX12-GI-NEXT: v_mul_f64_e32 v[8:9], 0x3df00000, v[4:5]
+; GFX12-GI-NEXT: v_mul_f64_e32 v[10:11], 0x3df00000, v[6:7]
+; GFX12-GI-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX12-GI-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX12-GI-NEXT: v_fma_f64 v[4:5], 0xc1f00000, v[8:9], v[4:5]
+; GFX12-GI-NEXT: v_fma_f64 v[6:7], 0xc1f00000, v[10:11], v[6:7]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v12, v[4:5]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v6, v[6:7]
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_mov_b32 v5, 0x43dfffff
+; GFX12-GI-NEXT: v_cvt_i32_f64_e32 v7, v[8:9]
+; GFX12-GI-NEXT: v_cvt_i32_f64_e32 v8, v[10:11]
+; GFX12-GI-NEXT: v_cmp_gt_f64_e64 s1, v[0:1], v[4:5]
+; GFX12-GI-NEXT: v_cmp_gt_f64_e64 s2, v[2:3], v[4:5]
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v9, v12, 0, vcc_lo
+; GFX12-GI-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v6, v6, 0, s0
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v7, 0x80000000, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v2, v8, 0x80000000, s0
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v4, v9, -1, s1
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v6, -1, s2
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v3, v0, 0x7fffffff, s1
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v5, v2, 0x7fffffff, s2
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v4, 0, s3
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v2, v1, 0, s4
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v3, 0, s3
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v3, v5, 0, s4
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i64> @llvm.fptosi.sat.v2f64.v2i64(<2 x double> %f)
ret <2 x i64> %x
}
@@ -3281,21 +3368,39 @@ define <4 x i1> @test_s_signed_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s1
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s0
-; GFX12-GI-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-GI-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s1
-; GFX12-GI-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v4, v0, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v5, v2, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX12-GI-NEXT: v_med3_i32 v0, v1, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v2, v4, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v1, v5, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v3, v3, -1, 0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-GI-NEXT: s_sext_i32_i16 s3, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s0, s0, 0
+; GFX12-GI-NEXT: s_min_i32 s2, s2, 0
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
+; GFX12-GI-NEXT: s_min_i32 s3, s3, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s3, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
+; GFX12-GI-NEXT: s_max_i32 s2, s2, -1
+; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
+; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptosi.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3541,35 +3646,39 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_lshr_b32 s2, s1, 16
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s2
-; GFX12-GI-NEXT: s_lshr_b32 s1, s0, 16
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-GI-NEXT: s_sext_i32_i16 s3, s3
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s1
-; GFX12-GI-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX12-GI-NEXT: s_movk_i32 s1, 0xff80
-; GFX12-GI-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s2, s2, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s3, s3, 0x7f
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_med3_i32 v0, v0, s1, 0x7f
-; GFX12-GI-NEXT: v_med3_i32 v1, v1, s1, 0x7f
-; GFX12-GI-NEXT: v_med3_i32 v2, v2, s1, 0x7f
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v4, 8, v2
-; GFX12-GI-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX12-GI-NEXT: v_med3_i32 v0, v3, s1, 0x7f
-; GFX12-GI-NEXT: v_and_b32_e32 v1, 0xff00, v4
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-GI-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-GI-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s3, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptosi.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3688,12 +3797,20 @@ define <4 x i16> @test_s_signed_v4f16_v4i16(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
-; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, s2
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, s3
+; GFX12-GI-NEXT: s_lshr_b32 s0, s1, 16
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s1, s2, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptosi.sat.v4f16.v4i16(<4 x half> %f)
ret <4 x i16> %x
@@ -5356,37 +5473,76 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s3
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s2
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s1
-; GFX12-GI-NEXT: s_lshr_b32 s3, s3, 16
-; GFX12-GI-NEXT: s_lshr_b32 s1, s1, 16
-; GFX12-GI-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v5.l, s1
-; GFX12-GI-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX12-GI-NEXT: v_med3_i32 v6, v0, -1, 0
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
; GFX12-GI-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-GI-NEXT: v_med3_i32 v4, v1, -1, 0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s1, 16
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s0
-; GFX12-GI-NEXT: s_lshr_b32 s0, s2, 16
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s3
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s0
+; GFX12-GI-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s4
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s6
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s0, s0, 0
+; GFX12-GI-NEXT: s_min_i32 s4, s6, 0
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s2
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s2, s6, 0
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_sext_i32_i16 s4, s4
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s5
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s5, s6, 0
+; GFX12-GI-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s5, s5
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0
+; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s3, s6
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
+; GFX12-GI-NEXT: s_max_i32 s4, s4, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s6
+; GFX12-GI-NEXT: s_sext_i32_i16 s7, s7
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0
+; GFX12-GI-NEXT: s_min_i32 s7, s7, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v7.l, s0
-; GFX12-GI-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v8, v1, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v9, v7, 0, 16
-; GFX12-GI-NEXT: v_med3_i32 v2, v2, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v0, v0, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v1, v3, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v7, v8, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v3, v5, -1, 0
-; GFX12-GI-NEXT: v_med3_i32 v5, v9, -1, 0
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s6
+; GFX12-GI-NEXT: s_sext_i32_i16 s7, s7
+; GFX12-GI-NEXT: s_max_i32 s2, s2, -1
+; GFX12-GI-NEXT: s_max_i32 s5, s5, -1
+; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
+; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s6, s6, -1
+; GFX12-GI-NEXT: s_max_i32 s7, s7, -1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5813,61 +5969,76 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s3
-; GFX12-GI-NEXT: s_lshr_b32 s4, s2, 16
-; GFX12-GI-NEXT: s_lshr_b32 s3, s3, 16
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s4
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_lshr_b32 s0, s0, 16
; GFX12-GI-NEXT: s_lshr_b32 s4, s1, 16
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s0
+; GFX12-GI-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s4
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s6
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s4, s6, 0x7f
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s2
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.l, s4
-; GFX12-GI-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v5.l, s3
-; GFX12-GI-NEXT: s_movk_i32 s2, 0xff80
-; GFX12-GI-NEXT: s_lshr_b32 s1, s0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX12-GI-NEXT: v_bfe_i32 v4, v4, 0, 16
+; GFX12-GI-NEXT: s_min_i32 s2, s6, 0x7f
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_sext_i32_i16 s4, s4
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s5
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s5, s6, 0x7f
+; GFX12-GI-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s5, s5
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0x7f
+; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s3, s6
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
+; GFX12-GI-NEXT: s_max_i32 s4, s4, 0xffffff80
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s1, s1
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s6
+; GFX12-GI-NEXT: s_sext_i32_i16 s7, s7
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s7, s7, 0x7f
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s6, s6
+; GFX12-GI-NEXT: s_sext_i32_i16 s7, s7
+; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s5, s5, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_med3_i32 v2, v2, s2, 0x7f
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v6.l, s1
-; GFX12-GI-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX12-GI-NEXT: v_med3_i32 v0, v0, s2, 0x7f
-; GFX12-GI-NEXT: v_med3_i32 v1, v1, s2, 0x7f
-; GFX12-GI-NEXT: v_med3_i32 v3, v3, s2, 0x7f
-; GFX12-GI-NEXT: v_med3_i32 v4, v4, s2, 0x7f
-; GFX12-GI-NEXT: v_and_b32_e32 v7, 0xff, v2
-; GFX12-GI-NEXT: v_bfe_i32 v2, v6, 0, 16
-; GFX12-GI-NEXT: v_med3_i32 v5, v5, s2, 0x7f
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-GI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v6.l, s0
-; GFX12-GI-NEXT: v_med3_i32 v9, v2, s2, 0x7f
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX12-GI-NEXT: v_or_b32_e32 v8, v0, v1
-; GFX12-GI-NEXT: v_or_b32_e32 v2, v3, v4
-; GFX12-GI-NEXT: v_bfe_i32 v0, v6, 0, 16
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v1, 8, v9
-; GFX12-GI-NEXT: v_or_b32_e32 v6, v7, v5
-; GFX12-GI-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-GI-NEXT: v_med3_i32 v0, v0, s2, 0x7f
-; GFX12-GI-NEXT: v_and_b32_e32 v5, 0xff00, v1
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v7, 16, v6
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-GI-NEXT: v_or_b32_e32 v9, v5, v4
-; GFX12-GI-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-GI-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX12-GI-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX12-GI-NEXT: v_mov_b32_e32 v4, v8
+; GFX12-GI-NEXT: s_max_i32 s6, s6, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s7, s7, 0xffffff80
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -6050,20 +6221,37 @@ define <8 x i16> @test_s_signed_v8f16_v8i16(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_lshr_b32 s4, s3, 16
-; GFX12-GI-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s1, 16
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, s4
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, s5
-; GFX12-GI-NEXT: s_lshr_b32 s4, s1, 16
-; GFX12-GI-NEXT: s_lshr_b32 s5, s0, 16
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s4
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s2, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s5, v2
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, s4
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, s5
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s4, v1
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s2
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.l, s1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v3
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v4
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s1, s5, s1
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s3, s6, s7
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i16> @llvm.fptosi.sat.v8f16.v8i16(<8 x half> %f)
ret <8 x i16> %x
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
index 93067ee1b46af..9ac190133aa85 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
@@ -167,24 +167,43 @@ define i32 @test_unsigned_i32_f32(float %f) nounwind {
}
define i64 @test_unsigned_i64_f32(float %f) nounwind {
-; GFX7-LABEL: test_unsigned_i64_f32:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_trunc_f32_e32 v1, v0
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
-; GFX7-NEXT: v_floor_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX7-NEXT: s_mov_b32 s4, 0xcf800000
-; GFX7-NEXT: v_fma_f32 v1, v2, s4, v1
-; GFX7-NEXT: v_cmp_nle_f32_e32 vcc, 0, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v3, 0, vcc
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX7-NEXT: s_mov_b32 s4, 0x5f7fffff
-; GFX7-NEXT: v_cmp_lt_f32_e64 s[4:5], s4, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v2, -1, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v3, 0, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_unsigned_i64_f32:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX7-ISEL-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
+; GFX7-ISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v3, v2
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0xcf800000
+; GFX7-ISEL-NEXT: v_fma_f32 v1, v2, s4, v1
+; GFX7-ISEL-NEXT: v_cmp_nle_f32_e32 vcc, 0, v0
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v3, 0, vcc
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0x5f7fffff
+; GFX7-ISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], s4, v0
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v1, v2, -1, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v3, 0, vcc
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_unsigned_i64_f32:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_trunc_f32_e32 v1, v0
+; GFX7-GI-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
+; GFX7-GI-NEXT: v_floor_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0xcf800000
+; GFX7-GI-NEXT: v_fma_f32 v1, v2, v3, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_cmp_nle_f32_e32 vcc, 0, v0
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0x5f7fffff
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX7-GI-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v0, v1, -1, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_unsigned_i64_f32:
; GFX9: ; %bb.0:
@@ -222,28 +241,51 @@ define i64 @test_unsigned_i64_f32(float %f) nounwind {
; GFX11-NEXT: v_cndmask_b32_e64 v0, v3, -1, vcc_lo
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_unsigned_i64_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_trunc_f32_e32 v1, v0
-; GFX12-NEXT: v_cmp_nle_f32_e32 vcc_lo, 0, v0
-; GFX12-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
-; GFX12-NEXT: v_floor_f32_e32 v2, v2
-; GFX12-NEXT: v_fmamk_f32 v1, v2, 0xcf800000, v1
-; GFX12-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX12-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v3, v1, 0, vcc_lo
-; GFX12-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x5f7fffff, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v3, -1, vcc_lo
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_unsigned_i64_f32:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX12-ISEL-NEXT: v_cmp_nle_f32_e32 vcc_lo, 0, v0
+; GFX12-ISEL-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
+; GFX12-ISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX12-ISEL-NEXT: v_fmamk_f32 v1, v2, 0xcf800000, v1
+; GFX12-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX12-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v3, v1, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x5f7fffff, v0
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v3, -1, vcc_lo
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_unsigned_i64_f32:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_trunc_f32_e32 v1, v0
+; GFX12-GI-NEXT: v_cmp_nle_f32_e32 vcc_lo, 0, v0
+; GFX12-GI-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
+; GFX12-GI-NEXT: v_floor_f32_e32 v2, v2
+; GFX12-GI-NEXT: v_fmac_f32_e32 v1, 0xcf800000, v2
+; GFX12-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX12-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc_lo
+; GFX12-GI-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x5f7fffff, v0
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v1, -1, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc_lo
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i64 @llvm.fptoui.sat.i64.f32(float %f)
ret i64 %x
}
@@ -705,27 +747,50 @@ define i32 @test_unsigned_i32_f64(double %f) nounwind {
}
define i64 @test_unsigned_i64_f64(double %f) nounwind {
-; GFX7-LABEL: test_unsigned_i64_f64:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX7-NEXT: s_movk_i32 s4, 0xffe0
-; GFX7-NEXT: s_mov_b32 s5, 0xc1f00000
-; GFX7-NEXT: v_cmp_nle_f64_e32 vcc, 0, v[0:1]
-; GFX7-NEXT: v_ldexp_f64 v[4:5], v[2:3], s4
-; GFX7-NEXT: s_mov_b32 s4, 0
-; GFX7-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
-; GFX7-NEXT: v_fma_f64 v[2:3], v[4:5], s[4:5], v[2:3]
-; GFX7-NEXT: s_mov_b32 s4, -1
-; GFX7-NEXT: s_mov_b32 s5, 0x43efffff
-; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], s[4:5], v[0:1]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v6, v[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v4, v6, 0, vcc
-; GFX7-NEXT: v_cvt_u32_f64_e32 v0, v[2:3]
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v4, -1, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_unsigned_i64_f64:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0xffe0
+; GFX7-ISEL-NEXT: s_mov_b32 s5, 0xc1f00000
+; GFX7-ISEL-NEXT: v_cmp_nle_f64_e32 vcc, 0, v[0:1]
+; GFX7-ISEL-NEXT: v_ldexp_f64 v[4:5], v[2:3], s4
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0
+; GFX7-ISEL-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX7-ISEL-NEXT: v_fma_f64 v[2:3], v[4:5], s[4:5], v[2:3]
+; GFX7-ISEL-NEXT: s_mov_b32 s4, -1
+; GFX7-ISEL-NEXT: s_mov_b32 s5, 0x43efffff
+; GFX7-ISEL-NEXT: v_cmp_lt_f64_e64 s[4:5], s[4:5], v[0:1]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v6, v[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v4, v6, 0, vcc
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v0, v[2:3]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v1, v4, -1, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_unsigned_i64_f64:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX7-GI-NEXT: v_mov_b32_e32 v4, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v5, 0x3df00000
+; GFX7-GI-NEXT: v_mov_b32_e32 v6, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v7, 0xc1f00000
+; GFX7-GI-NEXT: v_cmp_nle_f64_e32 vcc, 0, v[0:1]
+; GFX7-GI-NEXT: v_mul_f64 v[4:5], v[2:3], v[4:5]
+; GFX7-GI-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX7-GI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], v[2:3]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v6, v[2:3]
+; GFX7-GI-NEXT: v_mov_b32_e32 v2, -1
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0x43efffff
+; GFX7-GI-NEXT: v_cmp_gt_f64_e64 s[4:5], v[0:1], v[2:3]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v1, v[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v0, v6, -1, s[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, -1, s[4:5]
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_unsigned_i64_f64:
; GFX9: ; %bb.0:
@@ -768,31 +833,55 @@ define i64 @test_unsigned_i64_f64(double %f) nounwind {
; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, -1, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_unsigned_i64_f64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX12-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: s_mov_b32 s0, -1
-; GFX12-NEXT: s_mov_b32 s1, 0x43efffff
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmp_lt_f64_e64 s0, s[0:1], v[0:1]
-; GFX12-NEXT: v_ldexp_f64 v[4:5], v[2:3], 0xffffffe0
-; GFX12-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
-; GFX12-NEXT: v_fma_f64 v[2:3], 0xc1f00000, v[4:5], v[2:3]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v2, v[2:3]
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v3, v4, 0, vcc_lo
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v3, -1, s0
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, -1, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_unsigned_i64_f64:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX12-ISEL-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0, v[0:1]
+; GFX12-ISEL-NEXT: s_mov_b32 s0, -1
+; GFX12-ISEL-NEXT: s_mov_b32 s1, 0x43efffff
+; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-ISEL-NEXT: v_cmp_lt_f64_e64 s0, s[0:1], v[0:1]
+; GFX12-ISEL-NEXT: v_ldexp_f64 v[4:5], v[2:3], 0xffffffe0
+; GFX12-ISEL-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX12-ISEL-NEXT: v_fma_f64 v[2:3], 0xc1f00000, v[4:5], v[2:3]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v2, v[2:3]
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v3, v4, 0, vcc_lo
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v3, -1, s0
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, s0
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_unsigned_i64_f64:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX12-GI-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0, v[0:1]
+; GFX12-GI-NEXT: v_mul_f64_e32 v[4:5], 0x3df00000, v[2:3]
+; GFX12-GI-NEXT: v_floor_f64_e32 v[4:5], v[4:5]
+; GFX12-GI-NEXT: v_fma_f64 v[2:3], 0xc1f00000, v[4:5], v[2:3]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v6, v[2:3]
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, -1 :: v_dual_mov_b32 v3, 0x43efffff
+; GFX12-GI-NEXT: v_cmp_gt_f64_e64 s0, v[0:1], v[2:3]
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v4, 0, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v5, v6, 0, vcc_lo
+; GFX12-GI-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v5, -1, s0
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v1, -1, s0
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i64 @llvm.fptoui.sat.i64.f64(double %f)
ret i64 %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 8b64ca8e61d01..be34cf629cd5f 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -1578,39 +1578,74 @@ define <2 x i32> @test_unsigned_v2f64_v2i32_duplicate(<2 x double> %f) {
}
define <2 x i64> @test_unsigned_v2f64_v2i64(<2 x double> %f) {
-; GFX7-LABEL: test_unsigned_v2f64_v2i64:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
-; GFX7-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
-; GFX7-NEXT: s_movk_i32 s4, 0xffe0
-; GFX7-NEXT: s_mov_b32 s5, 0xc1f00000
-; GFX7-NEXT: s_mov_b32 s8, -1
-; GFX7-NEXT: s_mov_b32 s9, 0x43efffff
-; GFX7-NEXT: v_cmp_nle_f64_e64 s[6:7], 0, v[2:3]
-; GFX7-NEXT: v_cmp_nle_f64_e32 vcc, 0, v[0:1]
-; GFX7-NEXT: v_ldexp_f64 v[8:9], v[4:5], s4
-; GFX7-NEXT: v_ldexp_f64 v[10:11], v[6:7], s4
-; GFX7-NEXT: s_mov_b32 s4, 0
-; GFX7-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
-; GFX7-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
-; GFX7-NEXT: v_fma_f64 v[4:5], v[8:9], s[4:5], v[4:5]
-; GFX7-NEXT: v_fma_f64 v[6:7], v[10:11], s[4:5], v[6:7]
-; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], s[8:9], v[0:1]
-; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], s[8:9], v[2:3]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v13, v[10:11]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v12, v[8:9]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v13, 0, s[6:7]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v2, v[4:5]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v4, v[6:7]
-; GFX7-NEXT: v_cndmask_b32_e64 v8, v12, 0, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v3, v0, -1, s[8:9]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[6:7]
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v8, -1, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, -1, s[8:9]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_unsigned_v2f64_v2i64:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX7-ISEL-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0xffe0
+; GFX7-ISEL-NEXT: s_mov_b32 s5, 0xc1f00000
+; GFX7-ISEL-NEXT: s_mov_b32 s8, -1
+; GFX7-ISEL-NEXT: s_mov_b32 s9, 0x43efffff
+; GFX7-ISEL-NEXT: v_cmp_nle_f64_e64 s[6:7], 0, v[2:3]
+; GFX7-ISEL-NEXT: v_cmp_nle_f64_e32 vcc, 0, v[0:1]
+; GFX7-ISEL-NEXT: v_ldexp_f64 v[8:9], v[4:5], s4
+; GFX7-ISEL-NEXT: v_ldexp_f64 v[10:11], v[6:7], s4
+; GFX7-ISEL-NEXT: s_mov_b32 s4, 0
+; GFX7-ISEL-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX7-ISEL-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX7-ISEL-NEXT: v_fma_f64 v[4:5], v[8:9], s[4:5], v[4:5]
+; GFX7-ISEL-NEXT: v_fma_f64 v[6:7], v[10:11], s[4:5], v[6:7]
+; GFX7-ISEL-NEXT: v_cmp_lt_f64_e64 s[4:5], s[8:9], v[0:1]
+; GFX7-ISEL-NEXT: v_cmp_lt_f64_e64 s[8:9], s[8:9], v[2:3]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v13, v[10:11]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v12, v[8:9]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v13, 0, s[6:7]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v2, v[4:5]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v4, v[6:7]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v8, v12, 0, vcc
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v3, v0, -1, s[8:9]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[6:7]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v1, v8, -1, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, s[4:5]
+; GFX7-ISEL-NEXT: v_cndmask_b32_e64 v2, v2, -1, s[8:9]
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_unsigned_v2f64_v2i64:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX7-GI-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX7-GI-NEXT: v_mov_b32_e32 v8, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v9, 0x3df00000
+; GFX7-GI-NEXT: v_mov_b32_e32 v12, 0
+; GFX7-GI-NEXT: v_mov_b32_e32 v13, 0xc1f00000
+; GFX7-GI-NEXT: v_cmp_nle_f64_e64 s[6:7], 0, v[2:3]
+; GFX7-GI-NEXT: v_cmp_nle_f64_e32 vcc, 0, v[0:1]
+; GFX7-GI-NEXT: v_mul_f64 v[10:11], v[4:5], v[8:9]
+; GFX7-GI-NEXT: v_mul_f64 v[8:9], v[6:7], v[8:9]
+; GFX7-GI-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX7-GI-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX7-GI-NEXT: v_fma_f64 v[4:5], v[10:11], v[12:13], v[4:5]
+; GFX7-GI-NEXT: v_fma_f64 v[6:7], v[8:9], v[12:13], v[6:7]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v12, v[4:5]
+; GFX7-GI-NEXT: v_mov_b32_e32 v4, -1
+; GFX7-GI-NEXT: v_mov_b32_e32 v5, 0x43efffff
+; GFX7-GI-NEXT: v_cmp_gt_f64_e64 s[8:9], v[2:3], v[4:5]
+; GFX7-GI-NEXT: v_cmp_gt_f64_e64 s[4:5], v[0:1], v[4:5]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v6, v[6:7]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v3, v[10:11]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v4, v[8:9]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v7, v12, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v6, 0, s[6:7]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v2, v1, -1, s[8:9]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v3, 0, vcc
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v3, v4, 0, s[6:7]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v0, v7, -1, s[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v1, v1, -1, s[4:5]
+; GFX7-GI-NEXT: v_cndmask_b32_e64 v3, v3, -1, s[8:9]
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_unsigned_v2f64_v2i64:
; GFX9: ; %bb.0:
@@ -1677,43 +1712,79 @@ define <2 x i64> @test_unsigned_v2f64_v2i64(<2 x double> %f) {
; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, -1, s2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_unsigned_v2f64_v2i64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
-; GFX12-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
-; GFX12-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_cmp_nle_f64_e64 s0, 0, v[2:3]
-; GFX12-NEXT: s_mov_b32 s2, -1
-; GFX12-NEXT: s_mov_b32 s3, 0x43efffff
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmp_lt_f64_e64 s1, s[2:3], v[0:1]
-; GFX12-NEXT: v_cmp_lt_f64_e64 s2, s[2:3], v[2:3]
-; GFX12-NEXT: v_ldexp_f64 v[8:9], v[4:5], 0xffffffe0
-; GFX12-NEXT: v_ldexp_f64 v[10:11], v[6:7], 0xffffffe0
-; GFX12-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
-; GFX12-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
-; GFX12-NEXT: v_fma_f64 v[4:5], 0xc1f00000, v[8:9], v[4:5]
-; GFX12-NEXT: v_fma_f64 v[6:7], 0xc1f00000, v[10:11], v[6:7]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v8, v[8:9]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v9, v[10:11]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
-; GFX12-NEXT: v_cvt_u32_f64_e32 v5, v[6:7]
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v6, v8, 0, vcc_lo
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v9, 0, s0
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v6, -1, s1
-; GFX12-NEXT: v_cndmask_b32_e64 v3, v0, -1, s2
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc_lo
-; GFX12-NEXT: v_cndmask_b32_e64 v4, v5, 0, s0
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v2, -1, s1
-; GFX12-NEXT: v_cndmask_b32_e64 v2, v4, -1, s2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_unsigned_v2f64_v2i64:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX12-ISEL-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX12-ISEL-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0, v[0:1]
+; GFX12-ISEL-NEXT: v_cmp_nle_f64_e64 s0, 0, v[2:3]
+; GFX12-ISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-ISEL-NEXT: s_mov_b32 s3, 0x43efffff
+; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-ISEL-NEXT: v_cmp_lt_f64_e64 s1, s[2:3], v[0:1]
+; GFX12-ISEL-NEXT: v_cmp_lt_f64_e64 s2, s[2:3], v[2:3]
+; GFX12-ISEL-NEXT: v_ldexp_f64 v[8:9], v[4:5], 0xffffffe0
+; GFX12-ISEL-NEXT: v_ldexp_f64 v[10:11], v[6:7], 0xffffffe0
+; GFX12-ISEL-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX12-ISEL-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX12-ISEL-NEXT: v_fma_f64 v[4:5], 0xc1f00000, v[8:9], v[4:5]
+; GFX12-ISEL-NEXT: v_fma_f64 v[6:7], 0xc1f00000, v[10:11], v[6:7]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v8, v[8:9]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v9, v[10:11]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v4, v[4:5]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v5, v[6:7]
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v6, v8, 0, vcc_lo
+; GFX12-ISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v9, 0, s0
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v1, v6, -1, s1
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v3, v0, -1, s2
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc_lo
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v4, v5, 0, s0
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v0, v2, -1, s1
+; GFX12-ISEL-NEXT: v_cndmask_b32_e64 v2, v4, -1, s2
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_unsigned_v2f64_v2i64:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX12-GI-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX12-GI-NEXT: v_cmp_nle_f64_e32 vcc_lo, 0, v[0:1]
+; GFX12-GI-NEXT: v_cmp_nle_f64_e64 s0, 0, v[2:3]
+; GFX12-GI-NEXT: v_mul_f64_e32 v[8:9], 0x3df00000, v[4:5]
+; GFX12-GI-NEXT: v_mul_f64_e32 v[10:11], 0x3df00000, v[6:7]
+; GFX12-GI-NEXT: v_floor_f64_e32 v[8:9], v[8:9]
+; GFX12-GI-NEXT: v_floor_f64_e32 v[10:11], v[10:11]
+; GFX12-GI-NEXT: v_fma_f64 v[4:5], 0xc1f00000, v[8:9], v[4:5]
+; GFX12-GI-NEXT: v_fma_f64 v[6:7], 0xc1f00000, v[10:11], v[6:7]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v12, v[4:5]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v6, v[6:7]
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_mov_b32 v5, 0x43efffff
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v7, v[8:9]
+; GFX12-GI-NEXT: v_cvt_u32_f64_e32 v8, v[10:11]
+; GFX12-GI-NEXT: v_cmp_gt_f64_e64 s1, v[0:1], v[4:5]
+; GFX12-GI-NEXT: v_cmp_gt_f64_e64 s2, v[2:3], v[4:5]
+; GFX12-GI-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v9, v12, 0, vcc_lo
+; GFX12-GI-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v6, 0, s0
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v3, v7, 0, vcc_lo
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v4, v8, 0, s0
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v0, v9, -1, s1
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v2, v1, -1, s2
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v1, v3, -1, s1
+; GFX12-GI-NEXT: v_cndmask_b32_e64 v3, v4, -1, s2
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i64> @llvm.fptoui.sat.v2f64.v2i64(<2 x double> %f)
ret <2 x i64> %x
}
@@ -3082,21 +3153,29 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_lshr_b32 s2, s1, 16
-; GFX12-GI-NEXT: s_lshr_b32 s3, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GI-NEXT: v_and_b32_e32 v4, 0xffff, v2
-; GFX12-GI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX12-GI-NEXT: v_min_u32_e32 v2, 1, v1
-; GFX12-GI-NEXT: v_min_u32_e32 v1, 1, v4
-; GFX12-GI-NEXT: v_min_u32_e32 v3, 1, v3
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GI-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-GI-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX12-GI-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX12-GI-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
+; GFX12-GI-NEXT: s_min_u32 s2, s2, 1
+; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
+; GFX12-GI-NEXT: s_min_u32 s3, s3, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptoui.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3308,30 +3387,29 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_lshr_b32 s2, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s2
-; GFX12-GI-NEXT: s_lshr_b32 s1, s0, 16
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GI-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-GI-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX12-GI-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX12-GI-NEXT: s_and_b32 s3, 0xffff, s3
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX12-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-GI-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-GI-NEXT: v_min_u32_e32 v4, 0xff, v2
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-GI-NEXT: v_or_b32_e32 v2, v1, v0
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v1, 8, v4
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-GI-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX12-GI-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_min_u32 s2, s2, 0xff
+; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptoui.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3446,12 +3524,20 @@ define <4 x i16> @test_s_unsigned_v4f16_v4i16(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
-; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, s2
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, s3
+; GFX12-GI-NEXT: s_lshr_b32 s0, s1, 16
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s1, s2, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptoui.sat.v4f16.v4i16(<4 x half> %f)
ret <4 x i16> %x
@@ -5043,37 +5129,54 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_lshr_b32 s4, s3, 16
-; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s4
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_lshr_b32 s0, s0, 16
; GFX12-GI-NEXT: s_lshr_b32 s4, s2, 16
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s5
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s5, v1
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s4
-; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.l, s3
-; GFX12-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v6.l, s2
-; GFX12-GI-NEXT: v_min_u32_e32 v7, 1, v0
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_min_u32_e32 v3, 1, v2
-; GFX12-GI-NEXT: v_min_u32_e32 v5, 1, v1
+; GFX12-GI-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GI-NEXT: s_and_b32 s5, 0xffff, s5
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s4
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-GI-NEXT: v_and_b32_e32 v8, 0xffff, v6
-; GFX12-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX12-GI-NEXT: v_min_u32_e32 v6, 1, v4
-; GFX12-GI-NEXT: v_min_u32_e32 v4, 1, v8
-; GFX12-GI-NEXT: v_min_u32_e32 v1, 1, v1
-; GFX12-GI-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX12-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s2
+; GFX12-GI-NEXT: s_min_u32 s2, s5, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s5, s6, 1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: s_and_b32 s4, 0xffff, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s3, s4, 1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v2
+; GFX12-GI-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX12-GI-NEXT: s_and_b32 s4, 0xffff, s6
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
+; GFX12-GI-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s4, s4, 1
+; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
+; GFX12-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX12-GI-NEXT: s_min_u32 s7, s7, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s6, s6, 1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5447,54 +5550,54 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_lshr_b32 s0, s0, 16
; GFX12-GI-NEXT: s_lshr_b32 s4, s2, 16
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX12-GI-NEXT: s_lshr_b32 s3, s3, 16
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s2
-; GFX12-GI-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s4
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.l, s3
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v5.l, s1
-; GFX12-GI-NEXT: s_lshr_b32 s1, s0, 16
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-GI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s5, v1
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s4
+; GFX12-GI-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v6.l, s1
-; GFX12-GI-NEXT: v_min_u32_e32 v4, 0xff, v4
-; GFX12-GI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX12-GI-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX12-GI-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-GI-NEXT: v_min_u32_e32 v7, 0xff, v2
-; GFX12-GI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX12-GI-NEXT: v_min_u32_e32 v5, 0xff, v5
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX12-GI-NEXT: v_or_b32_e32 v8, v1, v0
-; GFX12-GI-NEXT: v_min_u32_e32 v1, 0xff, v6
-; GFX12-GI-NEXT: v_or_b32_e32 v6, v7, v4
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX12-GI-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v7, 16, v6
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-GI-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-GI-NEXT: v_or_b32_e32 v9, v1, v4
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-GI-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
-; GFX12-GI-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX12-GI-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX12-GI-NEXT: v_mov_b32_e32 v4, v8
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GI-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s1
+; GFX12-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s2
+; GFX12-GI-NEXT: s_min_u32 s2, s5, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s5, s6, 0xff
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GI-NEXT: s_and_b32 s4, 0xffff, s7
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s3, s4, 0xff
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v2
+; GFX12-GI-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX12-GI-NEXT: s_and_b32 s4, 0xffff, s6
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s4, s4, 0xff
+; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX12-GI-NEXT: s_min_u32 s7, s7, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s6, s6, 0xff
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -5671,20 +5774,37 @@ define <8 x i16> @test_s_unsigned_v8f16_v8i16(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_lshr_b32 s4, s3, 16
-; GFX12-GI-NEXT: s_lshr_b32 s5, s2, 16
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s1, 16
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, s4
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, s5
-; GFX12-GI-NEXT: s_lshr_b32 s4, s1, 16
-; GFX12-GI-NEXT: s_lshr_b32 s5, s0, 16
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s4
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s2, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s5, v2
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, s4
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, s5
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s1
+; GFX12-GI-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s4, v1
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s2
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.l, s1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s3, v2
+; GFX12-GI-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12-GI-NEXT: v_readfirstlane_b32 s6, v3
+; GFX12-GI-NEXT: v_readfirstlane_b32 s7, v4
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s1, s5, s1
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: s_pack_ll_b32_b16 s3, s6, s7
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i16> @llvm.fptoui.sat.v8f16.v8i16(<8 x half> %f)
ret <8 x i16> %x
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 16845ad9a501b..0c24ed011f028 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -38,19 +38,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; SI-GISEL-LABEL: fptrunc_f32_to_f16:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fptrunc_f32_to_f16:
@@ -74,19 +68,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; VI-GISEL-LABEL: fptrunc_f32_to_f16:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fptrunc_f32_to_f16:
@@ -110,19 +98,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; GFX9-GISEL-LABEL: fptrunc_f32_to_f16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fptrunc_f32_to_f16:
@@ -146,19 +128,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; GFX950-GISEL-LABEL: fptrunc_f32_to_f16:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16:
@@ -200,37 +176,25 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; GFX11-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16:
@@ -337,19 +301,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; SI-GISEL-LABEL: fptrunc_f32_to_f16_afn:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fptrunc_f32_to_f16_afn:
@@ -373,19 +331,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; VI-GISEL-LABEL: fptrunc_f32_to_f16_afn:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fptrunc_f32_to_f16_afn:
@@ -409,19 +361,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; GFX9-GISEL-LABEL: fptrunc_f32_to_f16_afn:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fptrunc_f32_to_f16_afn:
@@ -445,19 +391,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; GFX950-GISEL-LABEL: fptrunc_f32_to_f16_afn:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_afn:
@@ -499,37 +439,25 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; GFX11-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_afn:
@@ -2069,21 +1997,19 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
; GFX9-GISEL-LABEL: fptrunc_v2f32_to_v2f16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_nop 0
+; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fptrunc_v2f32_to_v2f16:
@@ -2160,41 +2086,37 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
; GFX11-GISEL-TRUE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[0:1], off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.h, v1
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, v0
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v1, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[0:1], off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_v2f32_to_v2f16:
@@ -4407,19 +4329,13 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; SI-GISEL-LABEL: fneg_fptrunc_f32_to_f16:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -v0
-; SI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fneg_fptrunc_f32_to_f16:
@@ -4443,19 +4359,13 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; VI-GISEL-LABEL: fneg_fptrunc_f32_to_f16:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -v0
-; VI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -s2
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fneg_fptrunc_f32_to_f16:
@@ -4479,19 +4389,13 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; GFX9-GISEL-LABEL: fneg_fptrunc_f32_to_f16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -v0
-; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -s2
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fneg_fptrunc_f32_to_f16:
@@ -4515,19 +4419,13 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; GFX950-GISEL-LABEL: fneg_fptrunc_f32_to_f16:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -v0
-; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -s2
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fneg_fptrunc_f32_to_f16:
@@ -4569,37 +4467,25 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; GFX11-GISEL-TRUE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, -v0
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, -s2
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, -v0
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, -s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fneg_fptrunc_f32_to_f16:
@@ -4713,19 +4599,13 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; SI-GISEL-LABEL: fabs_fptrunc_f32_to_f16:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; SI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fabs_fptrunc_f32_to_f16:
@@ -4749,19 +4629,13 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; VI-GISEL-LABEL: fabs_fptrunc_f32_to_f16:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; VI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fabs_fptrunc_f32_to_f16:
@@ -4785,19 +4659,13 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; GFX9-GISEL-LABEL: fabs_fptrunc_f32_to_f16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fabs_fptrunc_f32_to_f16:
@@ -4821,19 +4689,13 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; GFX950-GISEL-LABEL: fabs_fptrunc_f32_to_f16:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fabs_fptrunc_f32_to_f16:
@@ -4875,37 +4737,25 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; GFX11-GISEL-TRUE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, |v0|
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, |s2|
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fabs_fptrunc_f32_to_f16:
@@ -5019,19 +4869,13 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; SI-GISEL-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|v0|
-; SI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|s2|
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fneg_fabs_fptrunc_f32_to_f16:
@@ -5055,19 +4899,13 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; VI-GISEL-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|v0|
-; VI-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|s2|
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fneg_fabs_fptrunc_f32_to_f16:
@@ -5091,19 +4929,13 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; GFX9-GISEL-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|v0|
-; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|s2|
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fneg_fabs_fptrunc_f32_to_f16:
@@ -5127,19 +4959,13 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; GFX950-GISEL-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|v0|
-; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, -|s2|
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
@@ -5181,37 +5007,25 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; GFX11-GISEL-TRUE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, -|v0|
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, -|s2|
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, -|v0|
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, -|s2|
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 8995e75e8f9dd..f42bd116f3a91 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -6258,43 +6258,92 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v3, v9, v[6:7]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1200-LABEL: clpeak_imad_pat_i64:
-; GFX1200: ; %bb.0: ; %entry
-; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1200-NEXT: s_wait_expcnt 0x0
-; GFX1200-NEXT: s_wait_samplecnt 0x0
-; GFX1200-NEXT: s_wait_bvhcnt 0x0
-; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_mul_lo_u32 v7, v4, v3
-; GFX1200-NEXT: v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
-; GFX1200-NEXT: v_mul_lo_u32 v6, v5, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_add_co_u32 v4, vcc_lo, v0, v4
-; GFX1200-NEXT: v_add3_u32 v1, v1, v7, v6
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_mul_lo_u32 v6, v4, v3
-; GFX1200-NEXT: v_mad_co_u64_u32 v[3:4], null, v4, v2, 0
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v5, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_mul_lo_u32 v2, v5, v2
-; GFX1200-NEXT: v_add3_u32 v4, v4, v6, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_mul_lo_u32 v2, v3, v1
-; GFX1200-NEXT: v_mul_lo_u32 v5, v4, v0
-; GFX1200-NEXT: v_mad_co_u64_u32 v[0:1], null, v3, v0, v[3:4]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_add3_u32 v1, v5, v1, v2
-; GFX1200-NEXT: v_mul_lo_u32 v2, v0, v4
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_mul_lo_u32 v4, v1, v3
-; GFX1200-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v3, v[0:1]
-; GFX1200-NEXT: v_add3_u32 v1, v4, v1, v2
-; GFX1200-NEXT: s_setpc_b64 s[30:31]
+; GFX1200-SDAG-LABEL: clpeak_imad_pat_i64:
+; GFX1200-SDAG: ; %bb.0: ; %entry
+; GFX1200-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1200-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1
+; GFX1200-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v7, v4, v3
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v6, v5, v2
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, v4
+; GFX1200-SDAG-NEXT: v_add3_u32 v1, v1, v7, v6
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v6, v4, v3
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[3:4], null, v4, v2, 0
+; GFX1200-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v5, vcc_lo
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v2, v5, v2
+; GFX1200-SDAG-NEXT: v_add3_u32 v4, v4, v6, v2
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v2, v3, v1
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v5, v4, v0
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v3, v0, v[3:4]
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-NEXT: v_add3_u32 v1, v5, v1, v2
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v2, v0, v4
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v4, v1, v3
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v3, v[0:1]
+; GFX1200-SDAG-NEXT: v_add3_u32 v1, v4, v1, v2
+; GFX1200-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1200-GISEL-LABEL: clpeak_imad_pat_i64:
+; GFX1200-GISEL: ; %bb.0: ; %entry
+; GFX1200-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1200-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v5, v0, v3
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v6, v1, v2
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v7, v0, v2
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-NEXT: v_add3_u32 v4, v5, v6, v4
+; GFX1200-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v7, v0
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v4, v1, vcc_lo
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v5, v0, v2
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX1200-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v7, 1
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v4, vcc_lo
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v4, v0, v4
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-NEXT: v_add3_u32 v1, v3, v1, v5
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v3, v0, v2
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v5, v1, v2
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v2, v0, v2
+; GFX1200-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT: v_add3_u32 v3, v4, v5, v3
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v4, v2, v0
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v3, v3, v0
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-GISEL-NEXT: v_add3_u32 v1, v1, v3, v4
+; GFX1200-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-SDAG-LABEL: clpeak_imad_pat_i64:
; GFX1250-SDAG: ; %bb.0: ; %entry
@@ -7107,68 +7156,144 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v5
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1200-LABEL: clpeak_imad_pat_v2i64:
-; GFX1200: ; %bb.0: ; %entry
-; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1200-NEXT: s_wait_expcnt 0x0
-; GFX1200-NEXT: s_wait_samplecnt 0x0
-; GFX1200-NEXT: s_wait_bvhcnt 0x0
-; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v1, vcc_lo
-; GFX1200-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v3, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_mul_lo_u32 v12, v9, v4
-; GFX1200-NEXT: v_mul_lo_u32 v13, v8, v5
-; GFX1200-NEXT: v_mad_co_u64_u32 v[0:1], null, v8, v4, 0
-; GFX1200-NEXT: v_mul_lo_u32 v14, v11, v6
-; GFX1200-NEXT: v_mul_lo_u32 v15, v10, v7
-; GFX1200-NEXT: v_mad_co_u64_u32 v[2:3], null, v10, v6, 0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_add3_u32 v12, v1, v13, v12
-; GFX1200-NEXT: v_add_co_u32 v1, vcc_lo, v0, v8
-; GFX1200-NEXT: v_add3_u32 v13, v3, v15, v14
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v3, null, v12, v9, vcc_lo
-; GFX1200-NEXT: v_add_co_u32 v8, vcc_lo, v2, v10
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v9, null, v13, v11, vcc_lo
-; GFX1200-NEXT: v_mul_lo_u32 v10, v3, v4
-; GFX1200-NEXT: v_mul_lo_u32 v11, v1, v5
-; GFX1200-NEXT: v_mul_lo_u32 v7, v8, v7
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_mul_lo_u32 v9, v9, v6
-; GFX1200-NEXT: v_mad_co_u64_u32 v[5:6], null, v8, v6, 0
-; GFX1200-NEXT: v_mad_co_u64_u32 v[3:4], null, v1, v4, 0
-; GFX1200-NEXT: v_add3_u32 v6, v6, v7, v9
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_add3_u32 v4, v4, v11, v10
-; GFX1200-NEXT: v_mul_lo_u32 v11, v3, v12
-; GFX1200-NEXT: v_mul_lo_u32 v12, v5, v13
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_mul_lo_u32 v9, v6, v2
-; GFX1200-NEXT: v_mul_lo_u32 v10, v4, v0
-; GFX1200-NEXT: v_mad_co_u64_u32 v[0:1], null, v3, v0, v[3:4]
-; GFX1200-NEXT: v_mad_co_u64_u32 v[7:8], null, v5, v2, v[5:6]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_add3_u32 v1, v10, v1, v11
-; GFX1200-NEXT: v_add3_u32 v8, v9, v8, v12
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_mul_lo_u32 v4, v0, v4
-; GFX1200-NEXT: v_mul_lo_u32 v6, v7, v6
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_mul_lo_u32 v9, v1, v3
-; GFX1200-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v3, v[0:1]
-; GFX1200-NEXT: v_mul_lo_u32 v10, v8, v5
-; GFX1200-NEXT: v_mad_co_u64_u32 v[2:3], null, v7, v5, v[7:8]
-; GFX1200-NEXT: v_add3_u32 v1, v9, v1, v4
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-NEXT: v_add3_u32 v3, v10, v3, v6
-; GFX1200-NEXT: s_setpc_b64 s[30:31]
+; GFX1200-SDAG-LABEL: clpeak_imad_pat_v2i64:
+; GFX1200-SDAG: ; %bb.0: ; %entry
+; GFX1200-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1200-SDAG-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1
+; GFX1200-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v1, vcc_lo
+; GFX1200-SDAG-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1
+; GFX1200-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v3, vcc_lo
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v12, v9, v4
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v13, v8, v5
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v8, v4, 0
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v14, v11, v6
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v15, v10, v7
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v10, v6, 0
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT: v_add3_u32 v12, v1, v13, v12
+; GFX1200-SDAG-NEXT: v_add_co_u32 v1, vcc_lo, v0, v8
+; GFX1200-SDAG-NEXT: v_add3_u32 v13, v3, v15, v14
+; GFX1200-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT: v_add_co_ci_u32_e64 v3, null, v12, v9, vcc_lo
+; GFX1200-SDAG-NEXT: v_add_co_u32 v8, vcc_lo, v2, v10
+; GFX1200-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-NEXT: v_add_co_ci_u32_e64 v9, null, v13, v11, vcc_lo
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v10, v3, v4
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v11, v1, v5
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v7, v8, v7
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v9, v9, v6
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[5:6], null, v8, v6, 0
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[3:4], null, v1, v4, 0
+; GFX1200-SDAG-NEXT: v_add3_u32 v6, v6, v7, v9
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT: v_add3_u32 v4, v4, v11, v10
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v11, v3, v12
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v12, v5, v13
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v9, v6, v2
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v10, v4, v0
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v3, v0, v[3:4]
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[7:8], null, v5, v2, v[5:6]
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-NEXT: v_add3_u32 v1, v10, v1, v11
+; GFX1200-SDAG-NEXT: v_add3_u32 v8, v9, v8, v12
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v4, v0, v4
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v6, v7, v6
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v9, v1, v3
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v3, v[0:1]
+; GFX1200-SDAG-NEXT: v_mul_lo_u32 v10, v8, v5
+; GFX1200-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v7, v5, v[7:8]
+; GFX1200-SDAG-NEXT: v_add3_u32 v1, v9, v1, v4
+; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-SDAG-NEXT: v_add3_u32 v3, v10, v3, v6
+; GFX1200-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1200-GISEL-LABEL: clpeak_imad_pat_v2i64:
+; GFX1200-GISEL: ; %bb.0: ; %entry
+; GFX1200-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1200-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v8, v0, v4
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v9, v0, v5
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v10, v1, v4
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v11, v2, v6
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v12, v2, v7
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v13, v3, v6
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v14, v0, v4
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v15, v2, v6
+; GFX1200-GISEL-NEXT: v_add3_u32 v8, v9, v10, v8
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-GISEL-NEXT: v_add3_u32 v9, v12, v13, v11
+; GFX1200-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v14, v0
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v8, v1, vcc_lo
+; GFX1200-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v15, v2
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, v9, v3, vcc_lo
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v10, v0, v4
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v5, v0, v5
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v1, v1, v4
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v11, v2, v6
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v7, v2, v7
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v3, v3, v6
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX1200-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v14, 1
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
+; GFX1200-GISEL-NEXT: v_add3_u32 v1, v5, v1, v10
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v8, vcc_lo
+; GFX1200-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v15, 1
+; GFX1200-GISEL-NEXT: v_add3_u32 v3, v7, v3, v11
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v8, v0, v4
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v5, v0, v5
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v9, v1, v4
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v11, v2, v6
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v7, v2, v7
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v12, v3, v6
+; GFX1200-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v0, 1
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT: v_add3_u32 v4, v5, v9, v8
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v5, v2, v6
+; GFX1200-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1
+; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1200-GISEL-NEXT: v_add3_u32 v6, v7, v12, v11
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v7, v0, v10
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v4, v4, v10
+; GFX1200-GISEL-NEXT: v_mul_hi_u32 v8, v5, v2
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v3, v5, v3
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v6, v6, v2
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v0, v0, v10
+; GFX1200-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2
+; GFX1200-GISEL-NEXT: v_add3_u32 v1, v1, v4, v7
+; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-GISEL-NEXT: v_add3_u32 v3, v3, v6, v8
+; GFX1200-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-SDAG-LABEL: clpeak_imad_pat_v2i64:
; GFX1250-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/literal64.ll b/llvm/test/CodeGen/AMDGPU/literal64.ll
index ca35ea674f743..a60e536449b6e 100644
--- a/llvm/test/CodeGen/AMDGPU/literal64.ll
+++ b/llvm/test/CodeGen/AMDGPU/literal64.ll
@@ -295,20 +295,35 @@ define amdgpu_ps i64 @s_and_b64(i64 inreg %a) {
; No V_AND_B64 instruction, it has to be split
define amdgpu_ps void @v_and_b64(i64 %a, ptr addrspace(1) %out) {
-; GFX1250-LABEL: v_and_b64:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x12345678, v0
-; GFX1250-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: v_and_b64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x12345678, v0
+; GFX1250-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_and_b64:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: v_and_b32_e32 v1, 15, v1
-; GFX13-NEXT: v_and_b32_e32 v0, 0x12345678, v0
-; GFX13-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GFX13-NEXT: s_endpgm
+; GFX1250-GISEL-LABEL: v_and_b64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x12345678, v0
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX1250-GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_and_b64:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x12345678, v0
+; GFX13-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_and_b64:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x12345678, v0
+; GFX13-GISEL-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX13-GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX13-GISEL-NEXT: s_endpgm
%result = and i64 %a, 64729929336
store i64 %result, ptr addrspace(1) %out, align 8
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
index 8544595b30b77..e8c05bb8a807a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
@@ -28,23 +28,18 @@ define amdgpu_kernel void @div_fixup_f16(
;
; VI-GISEL-LABEL: div_fixup_f16:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL: s_mov_b32 s11, 0xf000
; VI-GISEL: s_mov_b32 s10, -1
-; VI-GISEL: s_mov_b32 s14, s10
-; VI-GISEL: s_mov_b32 s15, s11
-; VI-GISEL: s_mov_b32 s12, s2
-; VI-GISEL: s_mov_b32 s13, s3
-; VI-GISEL: s_mov_b32 s16, s4
-; VI-GISEL: s_mov_b32 s17, s5
-; VI-GISEL: s_mov_b32 s18, s10
-; VI-GISEL: s_mov_b32 s19, s11
-; VI-GISEL: s_mov_b32 s4, s6
-; VI-GISEL: s_mov_b32 s5, s7
-; VI-GISEL: s_mov_b32 s6, s10
-; VI-GISEL: s_mov_b32 s7, s11
-; VI-GISEL: s_mov_b32 s8, s0
-; VI-GISEL: s_mov_b32 s9, s1
-; VI-GISEL: v_div_fixup_f16 v0, v0, v1, v2
+; VI-GISEL: s_mov_b32 s11, 0xf000
+; VI-GISEL: s_mov_b64 s[8:9], s[2:3]
+; VI-GISEL: s_mov_b64 s[8:9], s[4:5]
+; VI-GISEL: s_mov_b64 s[8:9], s[6:7]
+; VI-GISEL: v_readfirstlane_b32 s2, v0
+; VI-GISEL: v_readfirstlane_b32 s3, v1
+; VI-GISEL: v_mov_b32_e32 v0, s3
+; VI-GISEL: v_readfirstlane_b32 s4, v2
+; VI-GISEL: v_mov_b32_e32 v1, s4
+; VI-GISEL: v_div_fixup_f16 v0, s2, v0, v1
+; VI-GISEL: s_mov_b64 s[2:3], s[10:11]
; VI-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
@@ -78,18 +73,17 @@ define amdgpu_kernel void @div_fixup_f16_imm_a(
;
; VI-GISEL-LABEL: div_fixup_f16_imm_a:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL: s_mov_b32 s7, 0xf000
; VI-GISEL: s_mov_b32 s6, -1
-; VI-GISEL: s_mov_b32 s14, s6
-; VI-GISEL: s_mov_b32 s12, s2
-; VI-GISEL: s_mov_b32 s13, s3
-; VI-GISEL: s_mov_b32 s15, s7
-; VI-GISEL: s_mov_b32 s10, s6
-; VI-GISEL: s_mov_b32 s11, s7
-; VI-GISEL: s_mov_b32 s4, s0
-; VI-GISEL: s_movk_i32 s0, 0x4200
-; VI-GISEL: s_mov_b32 s5, s1
-; VI-GISEL: v_div_fixup_f16 v0, s0, v0, v1
+; VI-GISEL: s_mov_b32 s7, 0xf000
+; VI-GISEL: s_mov_b64 s[10:11], s[6:7]
+; VI-GISEL: s_mov_b64 s[4:5], s[2:3]
+; VI-GISEL: v_mov_b32_e32 v2, 0x4200
+; VI-GISEL: v_readfirstlane_b32 s2, v0
+; VI-GISEL: v_readfirstlane_b32 s3, v1
+; VI-GISEL: v_mov_b32_e32 v0, s3
+; VI-GISEL: v_div_fixup_f16 v0, v2, s2, v0
+; VI-GISEL: s_mov_b64 s[2:3], s[6:7]
+; VI-GISEL: s_nop 1
; VI-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %b,
@@ -121,18 +115,17 @@ define amdgpu_kernel void @div_fixup_f16_imm_b(
;
; VI-GISEL-LABEL: div_fixup_f16_imm_b:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL: s_mov_b32 s7, 0xf000
; VI-GISEL: s_mov_b32 s6, -1
-; VI-GISEL: s_mov_b32 s14, s6
-; VI-GISEL: s_mov_b32 s12, s2
-; VI-GISEL: s_mov_b32 s13, s3
-; VI-GISEL: s_mov_b32 s15, s7
-; VI-GISEL: s_mov_b32 s10, s6
-; VI-GISEL: s_mov_b32 s11, s7
-; VI-GISEL: s_mov_b32 s4, s0
-; VI-GISEL: s_movk_i32 s0, 0x4200
-; VI-GISEL: s_mov_b32 s5, s1
-; VI-GISEL: v_div_fixup_f16 v0, v0, s0, v1
+; VI-GISEL: s_mov_b32 s7, 0xf000
+; VI-GISEL: s_mov_b64 s[10:11], s[6:7]
+; VI-GISEL: s_mov_b64 s[4:5], s[2:3]
+; VI-GISEL: v_mov_b32_e32 v2, 0x4200
+; VI-GISEL: v_readfirstlane_b32 s2, v0
+; VI-GISEL: v_readfirstlane_b32 s3, v1
+; VI-GISEL: v_mov_b32_e32 v0, s3
+; VI-GISEL: v_div_fixup_f16 v0, s2, v2, v0
+; VI-GISEL: s_mov_b64 s[2:3], s[6:7]
+; VI-GISEL: s_nop 1
; VI-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
@@ -164,18 +157,17 @@ define amdgpu_kernel void @div_fixup_f16_imm_c(
;
; VI-GISEL-LABEL: div_fixup_f16_imm_c:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL: s_mov_b32 s7, 0xf000
; VI-GISEL: s_mov_b32 s6, -1
-; VI-GISEL: s_mov_b32 s14, s6
-; VI-GISEL: s_mov_b32 s12, s2
-; VI-GISEL: s_mov_b32 s13, s3
-; VI-GISEL: s_mov_b32 s15, s7
-; VI-GISEL: s_mov_b32 s10, s6
-; VI-GISEL: s_mov_b32 s11, s7
-; VI-GISEL: s_mov_b32 s4, s0
-; VI-GISEL: s_movk_i32 s0, 0x4200
-; VI-GISEL: s_mov_b32 s5, s1
-; VI-GISEL: v_div_fixup_f16 v0, v0, v1, s0
+; VI-GISEL: s_mov_b32 s7, 0xf000
+; VI-GISEL: s_mov_b64 s[10:11], s[6:7]
+; VI-GISEL: s_mov_b64 s[4:5], s[2:3]
+; VI-GISEL: v_mov_b32_e32 v2, 0x4200
+; VI-GISEL: v_readfirstlane_b32 s2, v0
+; VI-GISEL: v_readfirstlane_b32 s3, v1
+; VI-GISEL: v_mov_b32_e32 v0, s3
+; VI-GISEL: v_div_fixup_f16 v0, s2, v0, v2
+; VI-GISEL: s_mov_b64 s[2:3], s[6:7]
+; VI-GISEL: s_nop 1
; VI-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
@@ -205,16 +197,14 @@ define amdgpu_kernel void @div_fixup_f16_imm_a_imm_b(
;
; VI-GISEL-LABEL: div_fixup_f16_imm_a_imm_b:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL: s_mov_b32 s7, 0xf000
; VI-GISEL: s_mov_b32 s6, -1
-; VI-GISEL: s_mov_b32 s10, s6
-; VI-GISEL: s_mov_b32 s11, s7
-; VI-GISEL: s_mov_b32 s8, s2
-; VI-GISEL: s_mov_b32 s9, s3
-; VI-GISEL: s_mov_b32 s4, s0
-; VI-GISEL: s_movk_i32 s0, 0x4200
-; VI-GISEL: s_mov_b32 s5, s1
-; VI-GISEL: v_div_fixup_f16 v0, s0, s0, v0
+; VI-GISEL: s_mov_b32 s7, 0xf000
+; VI-GISEL: v_mov_b32_e32 v1, 0x4200
+; VI-GISEL: s_mov_b64 s[4:5], s[2:3]
+; VI-GISEL: v_readfirstlane_b32 s2, v0
+; VI-GISEL: v_div_fixup_f16 v0, v1, v1, s2
+; VI-GISEL: s_mov_b64 s[2:3], s[6:7]
+; VI-GISEL: s_nop 2
; VI-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %c) {
@@ -242,16 +232,14 @@ define amdgpu_kernel void @div_fixup_f16_imm_b_imm_c(
;
; VI-GISEL-LABEL: div_fixup_f16_imm_b_imm_c:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL: s_mov_b32 s7, 0xf000
; VI-GISEL: s_mov_b32 s6, -1
-; VI-GISEL: s_mov_b32 s10, s6
-; VI-GISEL: s_mov_b32 s11, s7
-; VI-GISEL: s_mov_b32 s8, s2
-; VI-GISEL: s_mov_b32 s9, s3
-; VI-GISEL: s_mov_b32 s4, s0
-; VI-GISEL: s_movk_i32 s0, 0x4200
-; VI-GISEL: s_mov_b32 s5, s1
-; VI-GISEL: v_div_fixup_f16 v0, v0, s0, s0
+; VI-GISEL: s_mov_b32 s7, 0xf000
+; VI-GISEL: v_mov_b32_e32 v1, 0x4200
+; VI-GISEL: s_mov_b64 s[4:5], s[2:3]
+; VI-GISEL: v_readfirstlane_b32 s2, v0
+; VI-GISEL: v_div_fixup_f16 v0, s2, v1, v1
+; VI-GISEL: s_mov_b64 s[2:3], s[6:7]
+; VI-GISEL: s_nop 2
; VI-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
@@ -279,16 +267,14 @@ define amdgpu_kernel void @div_fixup_f16_imm_a_imm_c(
;
; VI-GISEL-LABEL: div_fixup_f16_imm_a_imm_c:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL: s_mov_b32 s7, 0xf000
; VI-GISEL: s_mov_b32 s6, -1
-; VI-GISEL: s_mov_b32 s10, s6
-; VI-GISEL: s_mov_b32 s11, s7
-; VI-GISEL: s_mov_b32 s8, s2
-; VI-GISEL: s_mov_b32 s9, s3
-; VI-GISEL: s_mov_b32 s4, s0
-; VI-GISEL: s_movk_i32 s0, 0x4200
-; VI-GISEL: s_mov_b32 s5, s1
-; VI-GISEL: v_div_fixup_f16 v0, s0, v0, s0
+; VI-GISEL: s_mov_b32 s7, 0xf000
+; VI-GISEL: v_mov_b32_e32 v1, 0x4200
+; VI-GISEL: s_mov_b64 s[4:5], s[2:3]
+; VI-GISEL: v_readfirstlane_b32 s2, v0
+; VI-GISEL: v_div_fixup_f16 v0, v1, s2, v1
+; VI-GISEL: s_mov_b64 s[2:3], s[6:7]
+; VI-GISEL: s_nop 2
; VI-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %b) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
index 1f534fa39b2b3..04b325b31f6a1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
@@ -183,7 +183,12 @@ define amdgpu_kernel void @mad_f16_neg_b(
; GFX9-GISEL-LABEL: mad_f16_neg_b:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL: v_mov_b32_e32 v0, 0
-; GFX9-GISEL: v_mad_legacy_f16 v1, v1, -v2, v3
+; GFX9-GISEL: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL: v_readfirstlane_b32 s1, v2
+; GFX9-GISEL: v_readfirstlane_b32 s2, v3
+; GFX9-GISEL: v_max_f16_e64 v1, s1, s1
+; GFX9-GISEL: v_mov_b32_e32 v2, s2
+; GFX9-GISEL: v_mad_legacy_f16 v1, s0, -v1, v2
; GFX9-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
@@ -264,7 +269,12 @@ define amdgpu_kernel void @mad_f16_neg_abs_b(
; GFX9-GISEL-LABEL: mad_f16_neg_abs_b:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL: v_mov_b32_e32 v0, 0
-; GFX9-GISEL: v_mad_legacy_f16 v1, v1, -|v2|, v3
+; GFX9-GISEL: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL: v_readfirstlane_b32 s1, v2
+; GFX9-GISEL: v_readfirstlane_b32 s2, v3
+; GFX9-GISEL: v_max_f16_e64 v1, |s1|, |s1|
+; GFX9-GISEL: v_mov_b32_e32 v2, s2
+; GFX9-GISEL: v_mad_legacy_f16 v1, s0, -v1, v2
; GFX9-GISEL: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
index f2b6095b35467..2fb67a86b7332 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
@@ -360,19 +360,32 @@ main_body:
}
define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #0 {
-; GFX11-TRUE16-LABEL: v_interp_f16_imm_params:
-; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_interp_p10_f16_f32 v2, v0.l, v2, v0.l wait_exp:7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_interp_p2_f16_f32 v0.l, v0.l, v3, v1 wait_exp:7
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-SDAG-LABEL: v_interp_f16_imm_params:
+; GFX11-TRUE16-SDAG: ; %bb.0: ; %main_body
+; GFX11-TRUE16-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GFX11-TRUE16-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX11-TRUE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-SDAG-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-SDAG-NEXT: v_interp_p10_f16_f32 v2, v0.l, v2, v0.l wait_exp:7
+; GFX11-TRUE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-SDAG-NEXT: v_interp_p2_f16_f32 v0.l, v0.l, v3, v1 wait_exp:7
+; GFX11-TRUE16-SDAG-NEXT: v_cvt_f16_f32_e32 v0.h, v2
+; GFX11-TRUE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-SDAG-NEXT: v_add_f16_e32 v0.l, v0.h, v0.l
+; GFX11-TRUE16-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-GISEL-LABEL: v_interp_f16_imm_params:
+; GFX11-TRUE16-GISEL: ; %bb.0: ; %main_body
+; GFX11-TRUE16-GISEL-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-GISEL-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX11-TRUE16-GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-TRUE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-GISEL-NEXT: v_interp_p10_f16_f32 v1, v0.l, v1, v0.l wait_exp:7
+; GFX11-TRUE16-GISEL-NEXT: v_interp_p2_f16_f32 v0.l, v0.l, v2, v3 wait_exp:7
+; GFX11-TRUE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-GISEL-NEXT: v_cvt_f16_f32_e32 v0.h, v1
+; GFX11-TRUE16-GISEL-NEXT: v_add_f16_e32 v0.l, v0.h, v0.l
+; GFX11-TRUE16-GISEL-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: v_interp_f16_imm_params:
; GFX11-FAKE16: ; %bb.0: ; %main_body
@@ -468,6 +481,3 @@ declare void @llvm.amdgcn.exp.f16(i32, i32, float, float, float, float, i1, i1)
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-TRUE16-GISEL: {{.*}}
-; GFX11-TRUE16-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index c22b8e9c6bc65..24bb3143c3b65 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -76,20 +76,22 @@ define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
; VARIANT2-GISEL: ; %bb.0: ; %entry
; VARIANT2-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VARIANT2-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VARIANT2-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VARIANT2-GISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; VARIANT2-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VARIANT2-GISEL-NEXT: global_store_dword v2, v0, s[0:1]
-; VARIANT2-GISEL-NEXT: v_xad_u32 v0, v0, -1, s2
+; VARIANT2-GISEL-NEXT: global_store_dword v4, v0, s[0:1]
+; VARIANT2-GISEL-NEXT: s_add_i32 s2, s2, -1
+; VARIANT2-GISEL-NEXT: v_sub_u32_e32 v0, s2, v0
; VARIANT2-GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VARIANT2-GISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
+; VARIANT2-GISEL-NEXT: v_mov_b32_e32 v2, s0
; VARIANT2-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; VARIANT2-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
+; VARIANT2-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; VARIANT2-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
; VARIANT2-GISEL-NEXT: s_waitcnt vmcnt(0)
; VARIANT2-GISEL-NEXT: s_barrier
; VARIANT2-GISEL-NEXT: global_load_dword v0, v[0:1], off
; VARIANT2-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VARIANT2-GISEL-NEXT: global_store_dword v2, v0, s[0:1]
+; VARIANT2-GISEL-NEXT: global_store_dword v4, v0, s[0:1]
; VARIANT2-GISEL-NEXT: s_endpgm
;
; VARIANT3-LABEL: test_barrier:
@@ -158,23 +160,23 @@ define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
; VARIANT6-LABEL: test_barrier:
; VARIANT6: ; %bb.0: ; %entry
; VARIANT6-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; VARIANT6-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; VARIANT6-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; VARIANT6-NEXT: v_lshlrev_b32_e32 v2, 2, v1
; VARIANT6-NEXT: s_wait_kmcnt 0x0
-; VARIANT6-NEXT: v_xad_u32 v0, v1, -1, s2
-; VARIANT6-NEXT: global_store_b32 v2, v1, s[0:1]
+; VARIANT6-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_and_b32 v1, 0x3ff, v0
+; VARIANT6-NEXT: s_add_co_i32 s2, s2, -1
+; VARIANT6-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; VARIANT6-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_lshlrev_b32 v4, 2, v1
+; VARIANT6-NEXT: v_sub_nc_u32_e32 v0, s2, v1
+; VARIANT6-NEXT: global_store_b32 v4, v1, s[0:1]
; VARIANT6-NEXT: s_barrier_signal -1
; VARIANT6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VARIANT6-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; VARIANT6-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; VARIANT6-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; VARIANT6-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; VARIANT6-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; VARIANT6-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; VARIANT6-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; VARIANT6-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
; VARIANT6-NEXT: s_barrier_wait -1
; VARIANT6-NEXT: global_load_b32 v0, v[0:1], off
; VARIANT6-NEXT: s_wait_loadcnt 0x0
-; VARIANT6-NEXT: global_store_b32 v2, v0, s[0:1]
+; VARIANT6-NEXT: global_store_b32 v4, v0, s[0:1]
; VARIANT6-NEXT: s_endpgm
entry:
%tmp = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll
index b6e7d956b421c..e5d423832148b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll
@@ -7,74 +7,142 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX900,GCN-GISEL,GFX900-GISEL %s
define double @v_exp_f64(double %in) #0 {
-; SI-LABEL: v_exp_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64:
; VI-SDAG: ; %bb.0:
@@ -316,104 +384,201 @@ define double @v_exp_f64(double %in) #0 {
}
define <2 x double> @v_exp_v2f64(<2 x double> %in) #0 {
-; SI-LABEL: v_exp_v2f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[4:5], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s26, -2
-; SI-NEXT: v_mov_b32_e32 v16, 0x43300000
-; SI-NEXT: v_bfi_b32 v7, s26, v16, v5
-; SI-NEXT: v_mov_b32_e32 v6, 0
-; SI-NEXT: v_add_f64 v[8:9], v[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_add_f64 v[7:8], v[8:9], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, s[6:7]
-; SI-NEXT: s_mov_b32 s8, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
-; SI-NEXT: s_mov_b32 s9, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[7:8], v[4:5], s[8:9], v[0:1]
-; SI-NEXT: v_mul_f64 v[14:15], v[2:3], s[4:5]
-; SI-NEXT: s_mov_b32 s10, 0x3b39803f
-; SI-NEXT: s_mov_b32 s11, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[8:9], v[4:5], s[10:11], v[7:8]
-; SI-NEXT: v_bfi_b32 v7, s26, v16, v15
-; SI-NEXT: v_add_f64 v[16:17], v[14:15], v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[6:7]
-; SI-NEXT: v_add_f64 v[6:7], v[16:17], -v[6:7]
-; SI-NEXT: s_mov_b32 s12, 0xfca7ab0c
-; SI-NEXT: s_mov_b32 s13, 0x3e928af3
-; SI-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc
-; SI-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc
-; SI-NEXT: s_mov_b32 s14, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s15, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v10, s12
-; SI-NEXT: v_mov_b32_e32 v11, s13
-; SI-NEXT: v_fma_f64 v[14:15], v[6:7], s[8:9], v[2:3]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], s[14:15], v[10:11]
-; SI-NEXT: s_mov_b32 s12, 0x623fde64
-; SI-NEXT: s_mov_b32 s13, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[14:15], v[6:7], s[10:11], v[14:15]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[12:13]
-; SI-NEXT: s_mov_b32 s16, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s17, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], s[14:15], v[10:11]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[16:17]
-; SI-NEXT: s_mov_b32 s18, 0x14761f6e
-; SI-NEXT: s_mov_b32 s19, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[12:13]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[18:19]
-; SI-NEXT: s_mov_b32 s20, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s21, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[16:17]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[20:21]
-; SI-NEXT: s_mov_b32 s22, 0x11122322
-; SI-NEXT: s_mov_b32 s23, 0x3f811111
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[18:19]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[22:23]
-; SI-NEXT: s_mov_b32 s24, 0x555502a1
-; SI-NEXT: s_mov_b32 s25, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[20:21]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[24:25]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[22:23]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[4:5]
-; SI-NEXT: s_mov_b32 s6, 11
-; SI-NEXT: s_mov_b32 s7, 0x3fe00000
-; SI-NEXT: v_cvt_i32_f64_e32 v16, v[4:5]
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[10:11], s[24:25]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[6:7]
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[4:5]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], 1.0
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[6:7]
-; SI-NEXT: v_fma_f64 v[8:9], v[8:9], v[12:13], 1.0
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x40900000
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
-; SI-NEXT: s_mov_b32 s8, 0
-; SI-NEXT: s_mov_b32 s9, 0xc090cc00
-; SI-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
-; SI-NEXT: v_ldexp_f64 v[8:9], v[8:9], v16
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[0:1]
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
-; SI-NEXT: v_mov_b32_e32 v7, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v9, v7, v9, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v9, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v8, vcc
-; SI-NEXT: v_ldexp_f64 v[4:5], v[4:5], v6
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[2:3]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v5, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v2f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[4:5], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s26, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v16, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v7, s26, v16, v5
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; SI-SDAG-NEXT: v_add_f64 v[8:9], v[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[7:8], v[8:9], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s8, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
+; SI-SDAG-NEXT: s_mov_b32 s9, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[4:5], s[8:9], v[0:1]
+; SI-SDAG-NEXT: v_mul_f64 v[14:15], v[2:3], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s10, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s11, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[4:5], s[10:11], v[7:8]
+; SI-SDAG-NEXT: v_bfi_b32 v7, s26, v16, v15
+; SI-SDAG-NEXT: v_add_f64 v[16:17], v[14:15], v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[16:17], -v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0xfca7ab0c
+; SI-SDAG-NEXT: s_mov_b32 s13, 0x3e928af3
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc
+; SI-SDAG-NEXT: s_mov_b32 s14, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, s12
+; SI-SDAG-NEXT: v_mov_b32_e32 v11, s13
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[6:7], s[8:9], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], s[14:15], v[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s13, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[6:7], s[10:11], v[14:15]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[12:13]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s17, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], s[14:15], v[10:11]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[22:23]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s6, 11
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3fe00000
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v16, v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[10:11], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[8:9], v[12:13], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x40900000
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s8, 0
+; SI-SDAG-NEXT: s_mov_b32 s9, 0xc090cc00
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
+; SI-SDAG-NEXT: v_ldexp_f64 v[8:9], v[8:9], v16
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v9, v7, v9, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v9, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v8, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[4:5], v[4:5], v6
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[2:3]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[2:3]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v5, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_v2f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[4:5], v[0:1], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v5
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_add_f64 v[10:11], v[4:5], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[9:10], v[10:11], -v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, -1
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, v[11:12]
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v7
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_add_f64 v[13:14], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[13:14], -v[8:9]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[6:7]|, v[11:12]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], -v[4:5], v[8:9], v[0:1]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], -v[6:7], v[8:9], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], -v[4:5], v[12:13], v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], -v[6:7], v[12:13], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[10:11], v[16:17], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x40900000
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0xc090cc00
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[14:15]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], v[0:1], v[16:17]
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[8:9], v[12:13], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[8:9], v[0:1], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v0, v[4:5]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v4, v[6:7]
+; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[10:11], v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x7ff00000
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[4:5], v[8:9], v4
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[2:3], v[14:15]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], v[2:3], v[16:17]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v10, v5, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_v2f64:
; VI-SDAG: ; %bb.0:
@@ -751,132 +916,258 @@ define <2 x double> @v_exp_v2f64(<2 x double> %in) #0 {
}
define <3 x double> @v_exp_v3f64(<3 x double> %in) #0 {
-; SI-LABEL: v_exp_v3f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s6, 0x652b82fe
-; SI-NEXT: s_mov_b32 s7, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[8:9], v[0:1], s[6:7]
-; SI-NEXT: s_brev_b32 s46, -2
-; SI-NEXT: v_mov_b32_e32 v10, 0x43300000
-; SI-NEXT: v_bfi_b32 v7, s46, v10, v9
-; SI-NEXT: v_mov_b32_e32 v6, 0
-; SI-NEXT: v_add_f64 v[11:12], v[8:9], v[6:7]
-; SI-NEXT: s_mov_b32 s8, -1
-; SI-NEXT: s_mov_b32 s9, 0x432fffff
-; SI-NEXT: v_add_f64 v[11:12], v[11:12], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[8:9]|, s[8:9]
-; SI-NEXT: s_mov_b32 s10, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v12, v12, v9, vcc
-; SI-NEXT: v_cndmask_b32_e32 v11, v11, v8, vcc
-; SI-NEXT: s_mov_b32 s11, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[7:8], v[11:12], s[10:11], v[0:1]
-; SI-NEXT: s_mov_b32 s12, 0x3b39803f
-; SI-NEXT: s_mov_b32 s13, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[13:14], v[11:12], s[12:13], v[7:8]
-; SI-NEXT: s_mov_b32 s4, 0xfca7ab0c
-; SI-NEXT: s_mov_b32 s5, 0x3e928af3
-; SI-NEXT: s_mov_b32 s14, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s15, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], s[14:15], v[8:9]
-; SI-NEXT: s_mov_b32 s16, 0x623fde64
-; SI-NEXT: s_mov_b32 s17, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[16:17]
-; SI-NEXT: s_mov_b32 s18, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s19, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[18:19]
-; SI-NEXT: s_mov_b32 s20, 0x14761f6e
-; SI-NEXT: s_mov_b32 s21, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[20:21]
-; SI-NEXT: s_mov_b32 s22, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s23, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[22:23]
-; SI-NEXT: s_mov_b32 s24, 0x11122322
-; SI-NEXT: s_mov_b32 s25, 0x3f811111
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[24:25]
-; SI-NEXT: s_mov_b32 s26, 0x555502a1
-; SI-NEXT: s_mov_b32 s27, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[26:27]
-; SI-NEXT: s_mov_b32 s28, 0x55555511
-; SI-NEXT: s_mov_b32 s29, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[28:29]
-; SI-NEXT: s_mov_b32 s42, 11
-; SI-NEXT: s_mov_b32 s43, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[42:43]
-; SI-NEXT: v_cvt_i32_f64_e32 v7, v[11:12]
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], 1.0
-; SI-NEXT: s_mov_b32 s40, 0
-; SI-NEXT: v_fma_f64 v[11:12], v[13:14], v[15:16], 1.0
-; SI-NEXT: v_mul_f64 v[13:14], v[2:3], s[6:7]
-; SI-NEXT: s_mov_b32 s41, 0x40900000
-; SI-NEXT: s_mov_b32 s44, 0
-; SI-NEXT: s_mov_b32 s45, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[11:12], v[11:12], v7
-; SI-NEXT: v_bfi_b32 v7, s46, v10, v14
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[44:45], v[0:1]
-; SI-NEXT: v_add_f64 v[15:16], v[13:14], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v19, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v12, v19, v12, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v12, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v11, vcc
-; SI-NEXT: v_add_f64 v[11:12], v[15:16], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[13:14]|, s[8:9]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[44:45], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc
-; SI-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
-; SI-NEXT: v_mul_f64 v[13:14], v[4:5], s[6:7]
-; SI-NEXT: v_fma_f64 v[15:16], v[11:12], s[10:11], v[2:3]
-; SI-NEXT: v_bfi_b32 v7, s46, v10, v14
-; SI-NEXT: v_add_f64 v[17:18], v[13:14], v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[13:14]|, s[8:9]
-; SI-NEXT: v_add_f64 v[6:7], v[17:18], -v[6:7]
-; SI-NEXT: v_fma_f64 v[15:16], v[11:12], s[12:13], v[15:16]
-; SI-NEXT: v_cndmask_b32_e32 v7, v7, v14, vcc
-; SI-NEXT: v_cndmask_b32_e32 v6, v6, v13, vcc
-; SI-NEXT: v_fma_f64 v[17:18], v[6:7], s[10:11], v[4:5]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], s[14:15], v[8:9]
-; SI-NEXT: v_fma_f64 v[17:18], v[6:7], s[12:13], v[17:18]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[16:17]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], s[14:15], v[8:9]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[18:19]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[16:17]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[20:21]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[18:19]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[22:23]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[20:21]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[24:25]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[22:23]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[26:27]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[24:25]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[28:29]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[26:27]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[42:43]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[28:29]
-; SI-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], 1.0
-; SI-NEXT: v_cvt_i32_f64_e32 v12, v[11:12]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[42:43]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[13:14], 1.0
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
-; SI-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
-; SI-NEXT: v_ldexp_f64 v[10:11], v[10:11], v12
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[2:3]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
-; SI-NEXT: v_cndmask_b32_e32 v7, v19, v11, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v7, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
-; SI-NEXT: v_ldexp_f64 v[6:7], v[8:9], v6
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[44:45], v[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v5, 0, v7, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v3f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[8:9], v[0:1], s[6:7]
+; SI-SDAG-NEXT: s_brev_b32 s46, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v7, s46, v10, v9
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; SI-SDAG-NEXT: v_add_f64 v[11:12], v[8:9], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s8, -1
+; SI-SDAG-NEXT: s_mov_b32 s9, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[11:12], v[11:12], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[8:9]|, s[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s10, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v12, v9, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v11, v11, v8, vcc
+; SI-SDAG-NEXT: s_mov_b32 s11, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[11:12], s[10:11], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s13, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[11:12], s[12:13], v[7:8]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfca7ab0c
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s14, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v9, s5
+; SI-SDAG-NEXT: v_mov_b32_e32 v8, s4
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], s[14:15], v[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s17, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[22:23]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s26, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s27, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[26:27]
+; SI-SDAG-NEXT: s_mov_b32 s28, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s29, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[28:29]
+; SI-SDAG-NEXT: s_mov_b32 s42, 11
+; SI-SDAG-NEXT: s_mov_b32 s43, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[42:43]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v7, v[11:12]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s40, 0
+; SI-SDAG-NEXT: v_fma_f64 v[11:12], v[13:14], v[15:16], 1.0
+; SI-SDAG-NEXT: v_mul_f64 v[13:14], v[2:3], s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s41, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s44, 0
+; SI-SDAG-NEXT: s_mov_b32 s45, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[11:12], v[11:12], v7
+; SI-SDAG-NEXT: v_bfi_b32 v7, s46, v10, v14
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[44:45], v[0:1]
+; SI-SDAG-NEXT: v_add_f64 v[15:16], v[13:14], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v19, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v19, v12, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v12, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v11, vcc
+; SI-SDAG-NEXT: v_add_f64 v[11:12], v[15:16], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[13:14]|, s[8:9]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[44:45], v[2:3]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
+; SI-SDAG-NEXT: v_mul_f64 v[13:14], v[4:5], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[11:12], s[10:11], v[2:3]
+; SI-SDAG-NEXT: v_bfi_b32 v7, s46, v10, v14
+; SI-SDAG-NEXT: v_add_f64 v[17:18], v[13:14], v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[13:14]|, s[8:9]
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[17:18], -v[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[11:12], s[12:13], v[15:16]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v14, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v13, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[17:18], v[6:7], s[10:11], v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], s[14:15], v[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[17:18], v[6:7], s[12:13], v[17:18]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], s[14:15], v[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[15:16], v[13:14], 1.0
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v12, v[11:12]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[13:14], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
+; SI-SDAG-NEXT: v_ldexp_f64 v[10:11], v[10:11], v12
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v19, v11, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v7, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[6:7], v[8:9], v6
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[44:45], v[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v7, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_v3f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[12:13], v[0:1], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v7, 0x80000000, v13
+; SI-GISEL-NEXT: v_or_b32_e32 v7, 0x43300000, v7
+; SI-GISEL-NEXT: v_add_f64 v[14:15], v[12:13], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[14:15], v[14:15], -v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[12:13]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v18, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v12, v14, v12, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v19, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v13, v15, v13, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], -v[12:13], v[18:19], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v20, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v21, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[16:17], v[2:3], v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], -v[12:13], v[20:21], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v24, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v25, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v26, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v27, 0x3e928af3
+; SI-GISEL-NEXT: v_and_b32_e32 v7, 0x80000000, v17
+; SI-GISEL-NEXT: v_fma_f64 v[28:29], v[14:15], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_or_b32_e32 v7, 0x43300000, v7
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, 0x3ec71dee
+; SI-GISEL-NEXT: v_add_f64 v[22:23], v[16:17], v[6:7]
+; SI-GISEL-NEXT: v_fma_f64 v[28:29], v[14:15], v[28:29], v[30:31]
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, 0x3efa0199
+; SI-GISEL-NEXT: v_add_f64 v[22:23], v[22:23], -v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[16:17]|, v[8:9]
+; SI-GISEL-NEXT: v_fma_f64 v[28:29], v[14:15], v[28:29], v[32:33]
+; SI-GISEL-NEXT: v_mov_b32_e32 v34, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v35, 0x3f2a01a0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v16, v22, v16, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v17, v23, v17, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[14:15], v[28:29], v[34:35]
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[14:15], v[22:23], v[28:29]
+; SI-GISEL-NEXT: v_mul_f64 v[10:11], v[4:5], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[14:15], v[22:23], v[36:37]
+; SI-GISEL-NEXT: v_and_b32_e32 v7, 0x80000000, v11
+; SI-GISEL-NEXT: v_mov_b32_e32 v38, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v39, 0x3fa55555
+; SI-GISEL-NEXT: v_or_b32_e32 v7, 0x43300000, v7
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[14:15], v[22:23], v[38:39]
+; SI-GISEL-NEXT: v_add_f64 v[48:49], v[10:11], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v50, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v51, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[14:15], v[22:23], v[50:51]
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[48:49], -v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v48, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v49, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[14:15], v[22:23], v[48:49]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[10:11]|, v[8:9]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[14:15], v[22:23], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], -v[16:17], v[18:19], v[2:3]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], -v[6:7], v[18:19], v[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], -v[16:17], v[20:21], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], -v[6:7], v[20:21], v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[18:19], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[18:19], v[24:25], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[18:19], v[24:25], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[34:35]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v26, v[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[18:19], v[24:25], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[20:21], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[18:19], v[24:25], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[24:25], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[48:49]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[8:9], v26
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v16, v[16:17]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0xc090cc00
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0x7ff00000
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v22, v17, v9, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[48:49]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v8, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[12:13], v16
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v22, vcc
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[2:3], v[10:11]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v16, v[6:7]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v12, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v13, v17, v9, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[18:19], v[20:21], 1.0
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[2:3], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[18:19], v[8:9], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v12, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[6:7], v[6:7], v16
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v13, vcc
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[4:5], v[10:11]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v17, v7, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[4:5], v[14:15]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v7, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_v3f64:
; VI-SDAG: ; %bb.0:
@@ -1306,160 +1597,315 @@ define <3 x double> @v_exp_v3f64(<3 x double> %in) #0 {
}
define <4 x double> @v_exp_v4f64(<4 x double> %in) #0 {
-; SI-LABEL: v_exp_v4f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s6, 0x652b82fe
-; SI-NEXT: s_mov_b32 s7, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[10:11], v[0:1], s[6:7]
-; SI-NEXT: s_brev_b32 s46, -2
-; SI-NEXT: v_mov_b32_e32 v18, 0x43300000
-; SI-NEXT: v_bfi_b32 v9, s46, v18, v11
-; SI-NEXT: v_mov_b32_e32 v8, 0
-; SI-NEXT: v_add_f64 v[12:13], v[10:11], v[8:9]
-; SI-NEXT: s_mov_b32 s8, -1
-; SI-NEXT: s_mov_b32 s9, 0x432fffff
-; SI-NEXT: v_mul_f64 v[14:15], v[2:3], s[6:7]
-; SI-NEXT: v_add_f64 v[12:13], v[12:13], -v[8:9]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[10:11]|, s[8:9]
-; SI-NEXT: v_bfi_b32 v9, s46, v18, v15
-; SI-NEXT: v_cndmask_b32_e32 v11, v13, v11, vcc
-; SI-NEXT: v_cndmask_b32_e32 v10, v12, v10, vcc
-; SI-NEXT: v_add_f64 v[12:13], v[14:15], v[8:9]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[8:9]
-; SI-NEXT: v_add_f64 v[12:13], v[12:13], -v[8:9]
-; SI-NEXT: s_mov_b32 s10, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
-; SI-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc
-; SI-NEXT: v_mul_f64 v[14:15], v[4:5], s[6:7]
-; SI-NEXT: s_mov_b32 s11, 0xbfe62e42
-; SI-NEXT: v_bfi_b32 v9, s46, v18, v15
-; SI-NEXT: v_add_f64 v[16:17], v[14:15], v[8:9]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[8:9]
-; SI-NEXT: v_add_f64 v[16:17], v[16:17], -v[8:9]
-; SI-NEXT: s_mov_b32 s12, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc
-; SI-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc
-; SI-NEXT: v_fma_f64 v[16:17], v[10:11], s[10:11], v[0:1]
-; SI-NEXT: s_mov_b32 s13, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[19:20], v[10:11], s[12:13], v[16:17]
-; SI-NEXT: s_mov_b32 s4, 0xfca7ab0c
-; SI-NEXT: s_mov_b32 s5, 0x3e928af3
-; SI-NEXT: s_mov_b32 s16, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s17, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v17, s5
-; SI-NEXT: v_mov_b32_e32 v16, s4
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], s[16:17], v[16:17]
-; SI-NEXT: s_mov_b32 s14, 0x623fde64
-; SI-NEXT: s_mov_b32 s15, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[14:15]
-; SI-NEXT: s_mov_b32 s18, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s19, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[18:19]
-; SI-NEXT: s_mov_b32 s20, 0x14761f6e
-; SI-NEXT: s_mov_b32 s21, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[20:21]
-; SI-NEXT: s_mov_b32 s22, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s23, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[22:23]
-; SI-NEXT: s_mov_b32 s24, 0x11122322
-; SI-NEXT: s_mov_b32 s25, 0x3f811111
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[24:25]
-; SI-NEXT: s_mov_b32 s40, 0x555502a1
-; SI-NEXT: s_mov_b32 s41, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[40:41]
-; SI-NEXT: s_mov_b32 s42, 0x55555511
-; SI-NEXT: s_mov_b32 s43, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[42:43]
-; SI-NEXT: s_mov_b32 s44, 11
-; SI-NEXT: s_mov_b32 s45, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[44:45]
-; SI-NEXT: v_cvt_i32_f64_e32 v9, v[10:11]
-; SI-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], 1.0
-; SI-NEXT: s_mov_b32 s28, 0
-; SI-NEXT: s_mov_b32 s29, 0x40900000
-; SI-NEXT: s_mov_b32 s26, 0
-; SI-NEXT: s_mov_b32 s27, 0xc090cc00
-; SI-NEXT: v_fma_f64 v[19:20], v[19:20], v[21:22], 1.0
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[0:1]
-; SI-NEXT: v_fma_f64 v[0:1], v[12:13], s[10:11], v[2:3]
-; SI-NEXT: v_ldexp_f64 v[9:10], v[19:20], v9
-; SI-NEXT: v_fma_f64 v[0:1], v[12:13], s[12:13], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v19, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v20, v19, v10, vcc
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], s[16:17], v[16:17]
-; SI-NEXT: v_cvt_i32_f64_e32 v12, v[12:13]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[14:15]
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[18:19]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[20:21]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[22:23]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[24:25]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[40:41]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[42:43]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[44:45]
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], 1.0
-; SI-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], 1.0
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v9, vcc
-; SI-NEXT: v_ldexp_f64 v[9:10], v[10:11], v12
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[2:3]
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v20, s[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v12, v19, v10, vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v12, s[4:5]
-; SI-NEXT: v_mul_f64 v[12:13], v[6:7], s[6:7]
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], s[10:11], v[4:5]
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, 0, v9, vcc
-; SI-NEXT: v_bfi_b32 v9, s46, v18, v13
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], s[12:13], v[10:11]
-; SI-NEXT: v_cvt_i32_f64_e32 v20, v[14:15]
-; SI-NEXT: v_add_f64 v[14:15], v[12:13], v[8:9]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[12:13]|, s[8:9]
-; SI-NEXT: v_add_f64 v[8:9], v[14:15], -v[8:9]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], s[16:17], v[16:17]
-; SI-NEXT: v_cndmask_b32_e32 v9, v9, v13, vcc
-; SI-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], s[10:11], v[6:7]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[14:15]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], s[12:13], v[12:13]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[18:19]
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], s[16:17], v[16:17]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[20:21]
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[14:15]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[22:23]
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[18:19]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[24:25]
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[20:21]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[40:41]
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[22:23]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[42:43]
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[24:25]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[44:45]
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[40:41]
-; SI-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], 1.0
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[42:43]
-; SI-NEXT: v_fma_f64 v[10:11], v[10:11], v[14:15], 1.0
-; SI-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[44:45]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
-; SI-NEXT: v_fma_f64 v[14:15], v[12:13], v[16:17], 1.0
-; SI-NEXT: v_ldexp_f64 v[10:11], v[10:11], v20
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[4:5]
-; SI-NEXT: v_fma_f64 v[12:13], v[12:13], v[14:15], 1.0
-; SI-NEXT: v_cndmask_b32_e32 v9, v19, v11, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v5, 0, v9, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v4, 0, v10, vcc
-; SI-NEXT: v_ldexp_f64 v[8:9], v[12:13], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[6:7]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[6:7]
-; SI-NEXT: v_cndmask_b32_e32 v9, v19, v9, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v7, 0, v9, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v4f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[10:11], v[0:1], s[6:7]
+; SI-SDAG-NEXT: s_brev_b32 s46, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v18, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v9, s46, v18, v11
+; SI-SDAG-NEXT: v_mov_b32_e32 v8, 0
+; SI-SDAG-NEXT: v_add_f64 v[12:13], v[10:11], v[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s8, -1
+; SI-SDAG-NEXT: s_mov_b32 s9, 0x432fffff
+; SI-SDAG-NEXT: v_mul_f64 v[14:15], v[2:3], s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[12:13], v[12:13], -v[8:9]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[10:11]|, s[8:9]
+; SI-SDAG-NEXT: v_bfi_b32 v9, s46, v18, v15
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v11, v13, v11, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v10, v12, v10, vcc
+; SI-SDAG-NEXT: v_add_f64 v[12:13], v[14:15], v[8:9]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[8:9]
+; SI-SDAG-NEXT: v_add_f64 v[12:13], v[12:13], -v[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s10, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc
+; SI-SDAG-NEXT: v_mul_f64 v[14:15], v[4:5], s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s11, 0xbfe62e42
+; SI-SDAG-NEXT: v_bfi_b32 v9, s46, v18, v15
+; SI-SDAG-NEXT: v_add_f64 v[16:17], v[14:15], v[8:9]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[8:9]
+; SI-SDAG-NEXT: v_add_f64 v[16:17], v[16:17], -v[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[10:11], s[10:11], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s13, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[19:20], v[10:11], s[12:13], v[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfca7ab0c
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s17, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v17, s5
+; SI-SDAG-NEXT: v_mov_b32_e32 v16, s4
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], s[16:17], v[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s14, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[22:23]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s40, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s41, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[40:41]
+; SI-SDAG-NEXT: s_mov_b32 s42, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s43, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[42:43]
+; SI-SDAG-NEXT: s_mov_b32 s44, 11
+; SI-SDAG-NEXT: s_mov_b32 s45, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], s[44:45]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v9, v[10:11]
+; SI-SDAG-NEXT: v_fma_f64 v[21:22], v[19:20], v[21:22], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s28, 0
+; SI-SDAG-NEXT: s_mov_b32 s29, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s26, 0
+; SI-SDAG-NEXT: s_mov_b32 s27, 0xc090cc00
+; SI-SDAG-NEXT: v_fma_f64 v[19:20], v[19:20], v[21:22], 1.0
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[12:13], s[10:11], v[2:3]
+; SI-SDAG-NEXT: v_ldexp_f64 v[9:10], v[19:20], v9
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[12:13], s[12:13], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v19, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v20, v19, v10, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], s[16:17], v[16:17]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v12, v[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[14:15]
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[40:41]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[0:1], v[10:11], 1.0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v9, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[9:10], v[10:11], v12
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[2:3]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v20, s[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[2:3]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v19, v10, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v12, s[4:5]
+; SI-SDAG-NEXT: v_mul_f64 v[12:13], v[6:7], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], s[10:11], v[4:5]
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v9, vcc
+; SI-SDAG-NEXT: v_bfi_b32 v9, s46, v18, v13
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], s[12:13], v[10:11]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v20, v[14:15]
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[12:13], v[8:9]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[12:13]|, s[8:9]
+; SI-SDAG-NEXT: v_add_f64 v[8:9], v[14:15], -v[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], s[16:17], v[16:17]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v13, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], s[10:11], v[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[14:15]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], s[12:13], v[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], s[16:17], v[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[14:15]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[40:41]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[40:41]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[10:11], v[14:15], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[10:11], v[14:15], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[12:13], v[16:17], s[44:45]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[12:13], v[16:17], 1.0
+; SI-SDAG-NEXT: v_ldexp_f64 v[10:11], v[10:11], v20
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[12:13], v[14:15], 1.0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v9, v19, v11, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v9, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v10, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[8:9], v[12:13], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[6:7]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[26:27], v[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v9, v19, v9, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v9, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_v4f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[14:15], v[0:1], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v15
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_add_f64 v[16:17], v[14:15], v[8:9]
+; SI-GISEL-NEXT: v_mul_f64 v[18:19], v[2:3], v[12:13]
+; SI-GISEL-NEXT: v_add_f64 v[16:17], v[16:17], -v[8:9]
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v19
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x432fffff
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, v[10:11]
+; SI-GISEL-NEXT: v_add_f64 v[20:21], v[18:19], v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v14, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v15, vcc
+; SI-GISEL-NEXT: v_add_f64 v[14:15], v[20:21], -v[8:9]
+; SI-GISEL-NEXT: v_mul_f64 v[20:21], v[4:5], v[12:13]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[18:19]|, v[10:11]
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v21
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_add_f64 v[22:23], v[20:21], v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v14, v14, v18, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v15, v15, v19, vcc
+; SI-GISEL-NEXT: v_add_f64 v[18:19], v[22:23], -v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v22, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v23, 0x3fe62e42
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[20:21]|, v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], -v[16:17], v[22:23], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v26, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v27, 0x3c7abc9e
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v19, v19, v21, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], -v[16:17], v[26:27], v[24:25]
+; SI-GISEL-NEXT: v_mov_b32_e32 v24, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v25, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[20:21], v[24:25], v[28:29]
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[20:21], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_mul_f64 v[12:13], v[6:7], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v34, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v35, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[20:21], v[30:31], v[34:35]
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v13
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, 0x3f2a01a0
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[20:21], v[30:31], v[36:37]
+; SI-GISEL-NEXT: v_add_f64 v[38:39], v[12:13], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v48, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v49, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[20:21], v[30:31], v[48:49]
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[38:39], -v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v38, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v39, 0x3f811111
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[12:13]|, v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[20:21], v[30:31], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[20:21], v[10:11], v[30:31]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v13, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[20:21], v[10:11], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v50, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v51, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[20:21], v[10:11], v[50:51]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v52, v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[20:21], v[10:11], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[20:21], v[10:11], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v20, v[16:17]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], -v[14:15], v[22:23], v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], -v[14:15], v[26:27], v[16:17]
+; SI-GISEL-NEXT: v_ldexp_f64 v[10:11], v[10:11], v20
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[24:25], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[16:17], v[20:21], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[16:17], v[20:21], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], -v[18:19], v[22:23], v[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], -v[8:9], v[22:23], v[6:7]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], -v[18:19], v[26:27], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], -v[8:9], v[26:27], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[26:27], v[20:21], v[24:25], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[24:25], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[26:27], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[24:25], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[24:25], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[24:25], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], v[48:49]
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x40900000
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], v[30:31]
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], v[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[24:25], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], v[50:51]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v53, 0, v10, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[24:25], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v10, v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[24:25], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[20:21], v[14:15], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v18, 0x7ff00000
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[24:25], v[12:13]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v19, v18, v11, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[10:11], v[14:15], v10
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[4:5], v[28:29]
+; SI-GISEL-NEXT: v_mov_b32_e32 v26, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v27, 0xc090cc00
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[50:51]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v11, v18, v11, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[26:27]
+; SI-GISEL-NEXT: v_ldexp_f64 v[16:17], v[16:17], v52
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 s[4:5], v[2:3], v[28:29]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v14, v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v53, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v19, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[2:3], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[22:23], v[12:13], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v17, v18, v17, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v16, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v17, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[4:5], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[22:23], v[8:9], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v10, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v11, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[8:9], v14
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[6:7], v[28:29]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v18, v9, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[6:7], v[26:27]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_v4f64:
; VI-SDAG: ; %bb.0:
@@ -4667,74 +5113,142 @@ define amdgpu_ps <8 x i32> @s_exp_v4f64(<4 x double> inreg %in) #0 {
}
define double @v_exp_fabs_f64(double %in) #0 {
-; SI-LABEL: v_exp_fabs_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_fabs_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_fabs_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], |v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], |v[0:1]|
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_fabs_f64:
; VI-SDAG: ; %bb.0:
@@ -4977,74 +5491,142 @@ define double @v_exp_fabs_f64(double %in) #0 {
}
define double @v_exp_fneg_fabs_f64(double %in) #0 {
-; SI-LABEL: v_exp_fneg_fabs_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0xbff71547
-; SI-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -|v[0:1]|
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc0900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x4090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_fneg_fabs_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -|v[0:1]|
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc0900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x4090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_fneg_fabs_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], -|v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], -|v[0:1]|
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, -|v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, -|v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_fneg_fabs_f64:
; VI-SDAG: ; %bb.0:
@@ -5288,74 +5870,142 @@ define double @v_exp_fneg_fabs_f64(double %in) #0 {
}
define double @v_exp_fneg_f64(double %in) #0 {
-; SI-LABEL: v_exp_fneg_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0xbff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc0900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x4090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_fneg_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc0900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x4090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_fneg_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], -v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], -v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, -v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, -v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_fneg_f64:
; VI-SDAG: ; %bb.0:
@@ -5598,68 +6248,130 @@ define double @v_exp_fneg_f64(double %in) #0 {
}
define double @v_exp_f64_fast(double %in) #0 {
-; SI-LABEL: v_exp_f64_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_fast:
; VI-SDAG: ; %bb.0:
@@ -5877,74 +6589,142 @@ define double @v_exp_f64_fast(double %in) #0 {
}
define double @v_exp_f64_afn(double %in) #0 {
-; SI-LABEL: v_exp_f64_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_afn:
; VI-SDAG: ; %bb.0:
@@ -6186,68 +6966,130 @@ define double @v_exp_f64_afn(double %in) #0 {
}
define double @v_exp_f64_ninf(double %in) #0 {
-; SI-LABEL: v_exp_f64_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_ninf:
; VI-SDAG: ; %bb.0:
@@ -6465,74 +7307,142 @@ define double @v_exp_f64_ninf(double %in) #0 {
}
define double @v_exp_f64_nnan(double %in) #0 {
-; SI-LABEL: v_exp_f64_nnan:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64_nnan:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_nnan:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_nnan:
; VI-SDAG: ; %bb.0:
@@ -6774,74 +7684,142 @@ define double @v_exp_f64_nnan(double %in) #0 {
}
define double @v_fabs_exp_f64_afn(double %in) #0 {
-; SI-LABEL: v_fabs_exp_f64_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_fabs_exp_f64_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_fabs_exp_f64_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], |v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], |v[0:1]|
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_fabs_exp_f64_afn:
; VI-SDAG: ; %bb.0:
@@ -7071,81 +8049,143 @@ define double @v_fabs_exp_f64_afn(double %in) #0 {
; GFX900-GISEL-NEXT: v_ldexp_f64 v[2:3], v[4:5], v2
; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
-; GFX900-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x7ff00000
-; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v2, s[4:5]
-; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
-; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
- %fabs = call double @llvm.fabs.f64(double %in)
- %result = call afn double @llvm.exp.f64(double %fabs)
- ret double %result
-}
-
-define double @v_exp_f64_nnan_ninf(double %in) #0 {
-; SI-LABEL: v_exp_f64_nnan_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x7ff00000
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v2, s[4:5]
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %fabs = call double @llvm.fabs.f64(double %in)
+ %result = call afn double @llvm.exp.f64(double %fabs)
+ ret double %result
+}
+
+define double @v_exp_f64_nnan_ninf(double %in) #0 {
+; SI-SDAG-LABEL: v_exp_f64_nnan_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_nnan_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_nnan_ninf:
; VI-SDAG: ; %bb.0:
@@ -7587,75 +8627,144 @@ define double @v_exp_f64_from_fpext_f16(half %src) #0 {
}
define double @v_exp_f64_from_fpext_f32(float %src) #0 {
-; SI-LABEL: v_exp_f64_from_fpext_f32:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[6:7]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64_from_fpext_f32:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_from_fpext_f32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[8:9], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], -v[2:3], v[10:11], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[2:3], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_from_fpext_f32:
; VI-SDAG: ; %bb.0:
@@ -8107,74 +9216,142 @@ define double @v_exp_f64_from_fpext_math_f16(half %src0, half %src1) #0 {
}
define double @v_exp_f64_contract(double %in) #0 {
-; SI-LABEL: v_exp_f64_contract:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64_contract:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_contract:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_contract:
; VI-SDAG: ; %bb.0:
@@ -8416,68 +9593,130 @@ define double @v_exp_f64_contract(double %in) #0 {
}
define double @v_exp_f64_contract_nnan_ninf(double %in) #0 {
-; SI-LABEL: v_exp_f64_contract_nnan_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x652b82fe
-; SI-NEXT: s_mov_b32 s5, 0x3ff71547
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0xbc7abc9e
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f64_contract_nnan_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x652b82fe
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ff71547
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbc7abc9e
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f64_contract_nnan_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x652b82fe
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3ff71547
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp_f64_contract_nnan_ninf:
; VI-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll
index 8f383b536e09f..a29e6be96b2e6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll
@@ -7,80 +7,154 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX900,GCN-GISEL,GFX900-GISEL %s
define double @v_exp10_f64(double %in) #0 {
-; SI-LABEL: v_exp10_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64:
; VI-SDAG: ; %bb.0:
@@ -346,112 +420,217 @@ define double @v_exp10_f64(double %in) #0 {
}
define <2 x double> @v_exp10_v2f64(<2 x double> %in) #0 {
-; SI-LABEL: v_exp10_v2f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[4:5], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s28, -2
-; SI-NEXT: v_mov_b32_e32 v16, 0x43300000
-; SI-NEXT: v_bfi_b32 v7, s28, v16, v5
-; SI-NEXT: v_mov_b32_e32 v6, 0
-; SI-NEXT: v_add_f64 v[8:9], v[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_add_f64 v[7:8], v[8:9], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, s[6:7]
-; SI-NEXT: s_mov_b32 s8, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
-; SI-NEXT: s_mov_b32 s9, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[7:8], v[4:5], s[8:9], v[0:1]
-; SI-NEXT: s_mov_b32 s10, 0xa994fd21
-; SI-NEXT: s_mov_b32 s11, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[7:8], v[4:5], s[10:11], v[7:8]
-; SI-NEXT: s_mov_b32 s12, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s13, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[9:10], v[7:8], s[12:13]
-; SI-NEXT: v_mul_f64 v[14:15], v[2:3], s[4:5]
-; SI-NEXT: s_mov_b32 s14, 0xbbb55516
-; SI-NEXT: s_mov_b32 s15, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[8:9], v[7:8], s[14:15], v[9:10]
-; SI-NEXT: v_bfi_b32 v7, s28, v16, v15
-; SI-NEXT: v_add_f64 v[16:17], v[14:15], v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[6:7]
-; SI-NEXT: v_add_f64 v[6:7], v[16:17], -v[6:7]
-; SI-NEXT: s_mov_b32 s16, 0xfca7ab0c
-; SI-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc
-; SI-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc
-; SI-NEXT: v_fma_f64 v[14:15], v[6:7], s[8:9], v[2:3]
-; SI-NEXT: s_mov_b32 s17, 0x3e928af3
-; SI-NEXT: v_fma_f64 v[14:15], v[6:7], s[10:11], v[14:15]
-; SI-NEXT: s_mov_b32 s18, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s19, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v10, s16
-; SI-NEXT: v_mov_b32_e32 v11, s17
-; SI-NEXT: v_mul_f64 v[16:17], v[14:15], s[12:13]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], s[18:19], v[10:11]
-; SI-NEXT: s_mov_b32 s16, 0x623fde64
-; SI-NEXT: s_mov_b32 s17, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[14:15], v[14:15], s[14:15], v[16:17]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[16:17]
-; SI-NEXT: s_mov_b32 s20, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s21, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], s[18:19], v[10:11]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[20:21]
-; SI-NEXT: s_mov_b32 s22, 0x14761f6e
-; SI-NEXT: s_mov_b32 s23, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[16:17]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[22:23]
-; SI-NEXT: s_mov_b32 s24, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s25, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[20:21]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[24:25]
-; SI-NEXT: s_mov_b32 s26, 0x11122322
-; SI-NEXT: s_mov_b32 s27, 0x3f811111
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[22:23]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[26:27]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[24:25]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[4:5]
-; SI-NEXT: s_mov_b32 s6, 0x55555511
-; SI-NEXT: s_mov_b32 s7, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[26:27]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[6:7]
-; SI-NEXT: s_mov_b32 s8, 11
-; SI-NEXT: s_mov_b32 s9, 0x3fe00000
-; SI-NEXT: v_cvt_i32_f64_e32 v16, v[4:5]
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[10:11], s[4:5]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[8:9]
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[6:7]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], 1.0
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[8:9]
-; SI-NEXT: v_fma_f64 v[8:9], v[8:9], v[12:13], 1.0
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x40900000
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
-; SI-NEXT: s_mov_b32 s8, 0
-; SI-NEXT: s_mov_b32 s9, 0xc090cc00
-; SI-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
-; SI-NEXT: v_ldexp_f64 v[8:9], v[8:9], v16
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[0:1]
-; SI-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
-; SI-NEXT: v_mov_b32_e32 v7, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v9, v7, v9, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v9, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v8, vcc
-; SI-NEXT: v_ldexp_f64 v[4:5], v[4:5], v6
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[2:3]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v5, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_v2f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[4:5], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s28, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v16, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v7, s28, v16, v5
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; SI-SDAG-NEXT: v_add_f64 v[8:9], v[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[7:8], v[8:9], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s8, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
+; SI-SDAG-NEXT: s_mov_b32 s9, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[4:5], s[8:9], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s10, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s11, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[4:5], s[10:11], v[7:8]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s13, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[9:10], v[7:8], s[12:13]
+; SI-SDAG-NEXT: v_mul_f64 v[14:15], v[2:3], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s14, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[7:8], s[14:15], v[9:10]
+; SI-SDAG-NEXT: v_bfi_b32 v7, s28, v16, v15
+; SI-SDAG-NEXT: v_add_f64 v[16:17], v[14:15], v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[14:15]|, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[16:17], -v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0xfca7ab0c
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[6:7], s[8:9], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s17, 0x3e928af3
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[6:7], s[10:11], v[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, s16
+; SI-SDAG-NEXT: v_mov_b32_e32 v11, s17
+; SI-SDAG-NEXT: v_mul_f64 v[16:17], v[14:15], s[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], s[18:19], v[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s17, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[14:15], s[14:15], v[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], s[18:19], v[10:11]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[22:23]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s26, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s27, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[26:27]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s8, 11
+; SI-SDAG-NEXT: s_mov_b32 s9, 0x3fe00000
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v16, v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[10:11], s[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], s[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[8:9], v[12:13], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x40900000
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s8, 0
+; SI-SDAG-NEXT: s_mov_b32 s9, 0xc090cc00
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
+; SI-SDAG-NEXT: v_ldexp_f64 v[8:9], v[8:9], v16
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[14:15], v[4:5], 1.0
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v9, v7, v9, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v9, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v8, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[4:5], v[4:5], v6
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[2:3]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[2:3]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v5, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_v2f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[4:5], v[0:1], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v5
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_add_f64 v[10:11], v[4:5], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[9:10], v[10:11], -v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, -1
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, v[11:12]
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v7
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_add_f64 v[13:14], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[13:14], -v[8:9]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[6:7]|, v[11:12]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0xa994fd21
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x509f79ff
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fd34413
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], -v[4:5], v[8:9], v[0:1]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], -v[6:7], v[8:9], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], -v[4:5], v[12:13], v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], -v[6:7], v[12:13], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[14:15], v[10:11], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0x40026bb1
+; SI-GISEL-NEXT: v_mul_f64 v[12:13], v[8:9], v[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[8:9], v[16:17], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[10:11], v[16:17], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x40900000
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0xc090cc00
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[14:15]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], v[0:1], v[16:17]
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[8:9], v[12:13], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[8:9], v[0:1], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v0, v[4:5]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v4, v[6:7]
+; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[10:11], v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x7ff00000
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[4:5], v[8:9], v4
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[2:3], v[14:15]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], v[2:3], v[16:17]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v10, v5, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_v2f64:
; VI-SDAG: ; %bb.0:
@@ -821,142 +1000,278 @@ define <2 x double> @v_exp10_v2f64(<2 x double> %in) #0 {
}
define <3 x double> @v_exp10_v3f64(<3 x double> %in) #0 {
-; SI-LABEL: v_exp10_v3f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, 0x979a371
-; SI-NEXT: s_mov_b32 s9, 0x400a934f
-; SI-NEXT: v_mul_f64 v[8:9], v[0:1], s[8:9]
-; SI-NEXT: s_brev_b32 s60, -2
-; SI-NEXT: v_mov_b32_e32 v10, 0x43300000
-; SI-NEXT: v_bfi_b32 v7, s60, v10, v9
-; SI-NEXT: v_mov_b32_e32 v6, 0
-; SI-NEXT: v_add_f64 v[11:12], v[8:9], v[6:7]
-; SI-NEXT: s_mov_b32 s10, -1
-; SI-NEXT: s_mov_b32 s11, 0x432fffff
-; SI-NEXT: v_add_f64 v[11:12], v[11:12], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[8:9]|, s[10:11]
-; SI-NEXT: s_mov_b32 s12, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v12, v12, v9, vcc
-; SI-NEXT: v_cndmask_b32_e32 v11, v11, v8, vcc
-; SI-NEXT: s_mov_b32 s13, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[7:8], v[11:12], s[12:13], v[0:1]
-; SI-NEXT: s_mov_b32 s14, 0xa994fd21
-; SI-NEXT: s_mov_b32 s15, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[7:8], v[11:12], s[14:15], v[7:8]
-; SI-NEXT: s_mov_b32 s16, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s17, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[13:14], v[7:8], s[16:17]
-; SI-NEXT: s_mov_b32 s18, 0xbbb55516
-; SI-NEXT: s_mov_b32 s19, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[13:14], v[7:8], s[18:19], v[13:14]
-; SI-NEXT: s_mov_b32 s4, 0xfca7ab0c
-; SI-NEXT: s_mov_b32 s5, 0x3e928af3
-; SI-NEXT: s_mov_b32 s20, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s21, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], s[20:21], v[8:9]
-; SI-NEXT: s_mov_b32 s22, 0x623fde64
-; SI-NEXT: s_mov_b32 s23, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[22:23]
-; SI-NEXT: s_mov_b32 s24, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s25, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[24:25]
-; SI-NEXT: s_mov_b32 s26, 0x14761f6e
-; SI-NEXT: s_mov_b32 s27, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[26:27]
-; SI-NEXT: s_mov_b32 s28, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s29, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[28:29]
-; SI-NEXT: s_mov_b32 s40, 0x11122322
-; SI-NEXT: s_mov_b32 s41, 0x3f811111
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[40:41]
-; SI-NEXT: s_mov_b32 s42, 0x555502a1
-; SI-NEXT: s_mov_b32 s43, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[42:43]
-; SI-NEXT: s_mov_b32 s44, 0x55555511
-; SI-NEXT: s_mov_b32 s45, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[44:45]
-; SI-NEXT: s_mov_b32 s56, 11
-; SI-NEXT: s_mov_b32 s57, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[56:57]
-; SI-NEXT: v_cvt_i32_f64_e32 v7, v[11:12]
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], 1.0
-; SI-NEXT: s_mov_b32 s46, 0
-; SI-NEXT: v_fma_f64 v[11:12], v[13:14], v[15:16], 1.0
-; SI-NEXT: v_mul_f64 v[13:14], v[2:3], s[8:9]
-; SI-NEXT: v_ldexp_f64 v[11:12], v[11:12], v7
-; SI-NEXT: v_bfi_b32 v7, s60, v10, v14
-; SI-NEXT: v_add_f64 v[15:16], v[13:14], v[6:7]
-; SI-NEXT: s_mov_b32 s47, 0x40900000
-; SI-NEXT: s_mov_b32 s58, 0
-; SI-NEXT: s_mov_b32 s59, 0xc090cc00
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[46:47], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[58:59], v[0:1]
-; SI-NEXT: v_add_f64 v[0:1], v[15:16], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[13:14]|, s[10:11]
-; SI-NEXT: v_mov_b32_e32 v21, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e64 v14, v1, v14, s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v13, v0, v13, s[6:7]
-; SI-NEXT: v_mul_f64 v[0:1], v[4:5], s[8:9]
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], s[12:13], v[2:3]
-; SI-NEXT: v_bfi_b32 v7, s60, v10, v1
-; SI-NEXT: v_add_f64 v[17:18], v[0:1], v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, s[10:11]
-; SI-NEXT: v_add_f64 v[6:7], v[17:18], -v[6:7]
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], s[14:15], v[15:16]
-; SI-NEXT: v_cndmask_b32_e64 v7, v7, v1, s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v6, v6, v0, s[6:7]
-; SI-NEXT: v_fma_f64 v[0:1], v[6:7], s[12:13], v[4:5]
-; SI-NEXT: v_mul_f64 v[17:18], v[15:16], s[16:17]
-; SI-NEXT: v_fma_f64 v[0:1], v[6:7], s[14:15], v[0:1]
-; SI-NEXT: v_fma_f64 v[15:16], v[15:16], s[18:19], v[17:18]
-; SI-NEXT: v_mul_f64 v[17:18], v[0:1], s[16:17]
-; SI-NEXT: v_cndmask_b32_e32 v12, v21, v12, vcc
-; SI-NEXT: v_fma_f64 v[17:18], v[0:1], s[18:19], v[17:18]
-; SI-NEXT: v_fma_f64 v[0:1], v[15:16], s[20:21], v[8:9]
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_fma_f64 v[19:20], v[15:16], v[0:1], s[22:23]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], s[20:21], v[8:9]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v11, vcc
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[19:20], s[24:25]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[22:23]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[26:27]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[24:25]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[28:29]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[26:27]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[40:41]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[28:29]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[42:43]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[40:41]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[44:45]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[42:43]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[56:57]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[44:45]
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v12, s[4:5]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], 1.0
-; SI-NEXT: v_cvt_i32_f64_e32 v12, v[13:14]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[56:57]
-; SI-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], 1.0
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
-; SI-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
-; SI-NEXT: v_ldexp_f64 v[10:11], v[10:11], v12
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[46:47], v[2:3]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[58:59], v[2:3]
-; SI-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
-; SI-NEXT: v_cndmask_b32_e32 v7, v21, v11, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v7, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
-; SI-NEXT: v_ldexp_f64 v[6:7], v[8:9], v6
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[46:47], v[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[58:59], v[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v7, v21, v7, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v5, 0, v7, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_v3f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s8, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s9, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[8:9], v[0:1], s[8:9]
+; SI-SDAG-NEXT: s_brev_b32 s60, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v7, s60, v10, v9
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; SI-SDAG-NEXT: v_add_f64 v[11:12], v[8:9], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s10, -1
+; SI-SDAG-NEXT: s_mov_b32 s11, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[11:12], v[11:12], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[8:9]|, s[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v12, v9, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v11, v11, v8, vcc
+; SI-SDAG-NEXT: s_mov_b32 s13, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[11:12], s[12:13], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s14, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[11:12], s[14:15], v[7:8]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s17, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[13:14], v[7:8], s[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[7:8], s[18:19], v[13:14]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfca7ab0c
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v9, s5
+; SI-SDAG-NEXT: v_mov_b32_e32 v8, s4
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], s[20:21], v[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[22:23]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s26, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s27, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[26:27]
+; SI-SDAG-NEXT: s_mov_b32 s28, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s29, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[28:29]
+; SI-SDAG-NEXT: s_mov_b32 s40, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s41, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[40:41]
+; SI-SDAG-NEXT: s_mov_b32 s42, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s43, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[42:43]
+; SI-SDAG-NEXT: s_mov_b32 s44, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s45, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[44:45]
+; SI-SDAG-NEXT: s_mov_b32 s56, 11
+; SI-SDAG-NEXT: s_mov_b32 s57, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[56:57]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v7, v[11:12]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s46, 0
+; SI-SDAG-NEXT: v_fma_f64 v[11:12], v[13:14], v[15:16], 1.0
+; SI-SDAG-NEXT: v_mul_f64 v[13:14], v[2:3], s[8:9]
+; SI-SDAG-NEXT: v_ldexp_f64 v[11:12], v[11:12], v7
+; SI-SDAG-NEXT: v_bfi_b32 v7, s60, v10, v14
+; SI-SDAG-NEXT: v_add_f64 v[15:16], v[13:14], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s47, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s58, 0
+; SI-SDAG-NEXT: s_mov_b32 s59, 0xc090cc00
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[46:47], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[58:59], v[0:1]
+; SI-SDAG-NEXT: v_add_f64 v[0:1], v[15:16], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[13:14]|, s[10:11]
+; SI-SDAG-NEXT: v_mov_b32_e32 v21, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v14, v1, v14, s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v13, v0, v13, s[6:7]
+; SI-SDAG-NEXT: v_mul_f64 v[0:1], v[4:5], s[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], s[12:13], v[2:3]
+; SI-SDAG-NEXT: v_bfi_b32 v7, s60, v10, v1
+; SI-SDAG-NEXT: v_add_f64 v[17:18], v[0:1], v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, s[10:11]
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[17:18], -v[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], s[14:15], v[15:16]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v7, v7, v1, s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v6, v6, v0, s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[6:7], s[12:13], v[4:5]
+; SI-SDAG-NEXT: v_mul_f64 v[17:18], v[15:16], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[6:7], s[14:15], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[15:16], s[18:19], v[17:18]
+; SI-SDAG-NEXT: v_mul_f64 v[17:18], v[0:1], s[16:17]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v21, v12, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[17:18], v[0:1], s[18:19], v[17:18]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[15:16], s[20:21], v[8:9]
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_fma_f64 v[19:20], v[15:16], v[0:1], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], s[20:21], v[8:9]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v11, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[19:20], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[40:41]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[40:41]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[44:45]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v12, s[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], 1.0
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v12, v[13:14]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[15:16], v[10:11], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
+; SI-SDAG-NEXT: v_ldexp_f64 v[10:11], v[10:11], v12
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[46:47], v[2:3]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[58:59], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[17:18], v[8:9], 1.0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v21, v11, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v7, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[6:7], v[8:9], v6
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[46:47], v[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[58:59], v[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v21, v7, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v7, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_v3f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[0:1], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v8, 0x80000000, v7
+; SI-GISEL-NEXT: v_or_b32_e32 v13, 0x43300000, v8
+; SI-GISEL-NEXT: v_add_f64 v[14:15], v[6:7], v[12:13]
+; SI-GISEL-NEXT: v_mul_f64 v[16:17], v[2:3], v[10:11]
+; SI-GISEL-NEXT: v_add_f64 v[14:15], v[14:15], -v[12:13]
+; SI-GISEL-NEXT: v_and_b32_e32 v13, 0x80000000, v17
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_or_b32_e32 v13, 0x43300000, v13
+; SI-GISEL-NEXT: v_add_f64 v[18:19], v[16:17], v[12:13]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[6:7]|, v[8:9]
+; SI-GISEL-NEXT: v_add_f64 v[18:19], v[18:19], -v[12:13]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v14, v14, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v15, v15, v7, vcc
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[16:17]|, v[8:9]
+; SI-GISEL-NEXT: v_mul_f64 v[10:11], v[4:5], v[10:11]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, v18, v16, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v19, v17, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0x509f79ff
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0x3fd34413
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], -v[14:15], v[16:17], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v20, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v21, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], -v[14:15], v[20:21], v[18:19]
+; SI-GISEL-NEXT: v_and_b32_e32 v13, 0x80000000, v11
+; SI-GISEL-NEXT: v_mov_b32_e32 v22, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v23, 0xbcaf48ad
+; SI-GISEL-NEXT: v_or_b32_e32 v13, 0x43300000, v13
+; SI-GISEL-NEXT: v_mul_f64 v[26:27], v[18:19], v[22:23]
+; SI-GISEL-NEXT: v_add_f64 v[24:25], v[10:11], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[18:19], v[28:29], v[26:27]
+; SI-GISEL-NEXT: v_add_f64 v[12:13], v[24:25], -v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v24, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v25, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v26, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v27, 0x3e928af3
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[10:11]|, v[8:9]
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[18:19], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, 0x3ec71dee
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, v12, v10, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v13, v11, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], -v[6:7], v[16:17], v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], -v[8:9], v[16:17], v[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], -v[6:7], v[20:21], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], -v[8:9], v[20:21], v[16:17]
+; SI-GISEL-NEXT: v_mul_f64 v[48:49], v[30:31], v[22:23]
+; SI-GISEL-NEXT: v_mul_f64 v[22:23], v[16:17], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[30:31], v[28:29], v[48:49]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], v[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[16:17], v[28:29], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[28:29], v[30:31], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_mov_b32_e32 v34, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v35, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[16:17], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[26:27], v[30:31], v[28:29], v[32:33]
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[16:17], v[24:25], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[26:27], v[30:31], v[26:27], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v38, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v39, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[16:17], v[24:25], v[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[30:31], v[26:27], v[34:35]
+; SI-GISEL-NEXT: v_mov_b32_e32 v50, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v51, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[30:31], v[24:25], v[36:37]
+; SI-GISEL-NEXT: v_mov_b32_e32 v20, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v21, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], v[20:21]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v26, v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[30:31], v[24:25], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v48, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v49, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[16:17], v[12:13], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[30:31], v[14:15], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[16:17], v[12:13], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[30:31], v[14:15], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[16:17], v[12:13], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[18:19], v[10:11], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[30:31], v[14:15], v[48:49]
+; SI-GISEL-NEXT: v_mov_b32_e32 v18, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v19, 0x40900000
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[16:17], v[12:13], v[50:51]
+; SI-GISEL-NEXT: v_ldexp_f64 v[10:11], v[10:11], v26
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[30:31], v[14:15], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v22, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v23, 0xc090cc00
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[16:17], v[12:13], v[20:21]
+; SI-GISEL-NEXT: v_mov_b32_e32 v20, 0x7ff00000
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[30:31], v[14:15], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v11, v20, v11, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[16:17], v[12:13], v[48:49]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v10, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[6:7], v[14:15], v6
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v11, vcc
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[2:3], v[18:19]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v11, v20, v7, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[16:17], v[12:13], 1.0
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[2:3], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[16:17], v[6:7], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[6:7], v[6:7], v8
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v11, vcc
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[4:5], v[18:19]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v20, v7, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[4:5], v[22:23]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v7, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_v3f64:
; VI-SDAG: ; %bb.0:
@@ -1426,172 +1741,339 @@ define <3 x double> @v_exp10_v3f64(<3 x double> %in) #0 {
}
define <4 x double> @v_exp10_v4f64(<4 x double> %in) #0 {
-; SI-LABEL: v_exp10_v4f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[8:9], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s8, -2
-; SI-NEXT: v_mov_b32_e32 v16, 0x43300000
-; SI-NEXT: v_bfi_b32 v15, s8, v16, v9
-; SI-NEXT: v_mov_b32_e32 v14, 0
-; SI-NEXT: v_add_f64 v[10:11], v[8:9], v[14:15]
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_add_f64 v[10:11], v[10:11], -v[14:15]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[8:9]|, s[6:7]
-; SI-NEXT: s_mov_b32 s16, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; SI-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; SI-NEXT: v_mul_f64 v[10:11], v[2:3], s[4:5]
-; SI-NEXT: s_mov_b32 s17, 0xbfd34413
-; SI-NEXT: v_bfi_b32 v15, s8, v16, v11
-; SI-NEXT: v_add_f64 v[12:13], v[10:11], v[14:15]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[10:11]|, s[6:7]
-; SI-NEXT: v_add_f64 v[12:13], v[12:13], -v[14:15]
-; SI-NEXT: s_mov_b32 s14, 0xa994fd21
-; SI-NEXT: v_cndmask_b32_e32 v11, v13, v11, vcc
-; SI-NEXT: v_cndmask_b32_e32 v10, v12, v10, vcc
-; SI-NEXT: v_mul_f64 v[12:13], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s15, 0x3c49dc1d
-; SI-NEXT: v_bfi_b32 v15, s8, v16, v13
-; SI-NEXT: v_add_f64 v[17:18], v[12:13], v[14:15]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[12:13]|, s[6:7]
-; SI-NEXT: v_add_f64 v[17:18], v[17:18], -v[14:15]
-; SI-NEXT: s_mov_b32 s20, 0x494ea3e9
-; SI-NEXT: v_cndmask_b32_e32 v13, v18, v13, vcc
-; SI-NEXT: v_cndmask_b32_e32 v12, v17, v12, vcc
-; SI-NEXT: v_mul_f64 v[17:18], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s21, 0xbcaf48ad
-; SI-NEXT: v_bfi_b32 v15, s8, v16, v18
-; SI-NEXT: v_add_f64 v[19:20], v[17:18], v[14:15]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[17:18]|, s[6:7]
-; SI-NEXT: v_add_f64 v[14:15], v[19:20], -v[14:15]
-; SI-NEXT: s_mov_b32 s22, 0xbbb55516
-; SI-NEXT: v_cndmask_b32_e32 v14, v14, v17, vcc
-; SI-NEXT: v_fma_f64 v[16:17], v[8:9], s[16:17], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v15, v15, v18, vcc
-; SI-NEXT: v_fma_f64 v[16:17], v[8:9], s[14:15], v[16:17]
-; SI-NEXT: s_mov_b32 s23, 0x40026bb1
-; SI-NEXT: v_mul_f64 v[18:19], v[16:17], s[20:21]
-; SI-NEXT: s_mov_b32 s4, 0xfca7ab0c
-; SI-NEXT: v_fma_f64 v[18:19], v[16:17], s[22:23], v[18:19]
-; SI-NEXT: s_mov_b32 s5, 0x3e928af3
-; SI-NEXT: s_mov_b32 s10, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s11, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v17, s5
-; SI-NEXT: v_mov_b32_e32 v16, s4
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], s[10:11], v[16:17]
-; SI-NEXT: s_mov_b32 s12, 0x623fde64
-; SI-NEXT: s_mov_b32 s13, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[12:13]
-; SI-NEXT: s_mov_b32 s18, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s19, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[18:19]
-; SI-NEXT: s_mov_b32 s24, 0x14761f6e
-; SI-NEXT: s_mov_b32 s25, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[24:25]
-; SI-NEXT: s_mov_b32 s26, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s27, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[26:27]
-; SI-NEXT: s_mov_b32 s42, 0x11122322
-; SI-NEXT: s_mov_b32 s43, 0x3f811111
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[42:43]
-; SI-NEXT: s_mov_b32 s44, 0x555502a1
-; SI-NEXT: s_mov_b32 s45, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[44:45]
-; SI-NEXT: s_mov_b32 s46, 0x55555511
-; SI-NEXT: s_mov_b32 s47, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[46:47]
-; SI-NEXT: s_mov_b32 s56, 11
-; SI-NEXT: s_mov_b32 s57, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[56:57]
-; SI-NEXT: s_mov_b32 s28, 0
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], 1.0
-; SI-NEXT: s_mov_b32 s29, 0x40900000
-; SI-NEXT: v_fma_f64 v[18:19], v[18:19], v[20:21], 1.0
-; SI-NEXT: v_fma_f64 v[20:21], v[10:11], s[16:17], v[2:3]
-; SI-NEXT: s_mov_b32 s40, 0
-; SI-NEXT: v_fma_f64 v[20:21], v[10:11], s[14:15], v[20:21]
-; SI-NEXT: s_mov_b32 s41, 0xc090cc00
-; SI-NEXT: v_mul_f64 v[22:23], v[20:21], s[20:21]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[28:29], v[0:1]
-; SI-NEXT: v_fma_f64 v[20:21], v[20:21], s[22:23], v[22:23]
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[40:41], v[0:1]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], s[10:11], v[16:17]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[6:7], s[28:29], v[2:3]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[12:13]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[8:9], s[40:41], v[2:3]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[18:19]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[24:25]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[26:27]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[42:43]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[44:45]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[46:47]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[56:57]
-; SI-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], 1.0
-; SI-NEXT: v_fma_f64 v[20:21], v[20:21], v[0:1], 1.0
-; SI-NEXT: v_fma_f64 v[0:1], v[12:13], s[16:17], v[4:5]
-; SI-NEXT: v_fma_f64 v[0:1], v[12:13], s[14:15], v[0:1]
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[20:21]
-; SI-NEXT: v_fma_f64 v[0:1], v[0:1], s[22:23], v[2:3]
-; SI-NEXT: v_fma_f64 v[2:3], v[14:15], s[16:17], v[6:7]
-; SI-NEXT: v_fma_f64 v[2:3], v[14:15], s[14:15], v[2:3]
-; SI-NEXT: s_and_b64 s[14:15], vcc, s[4:5]
-; SI-NEXT: v_mul_f64 v[22:23], v[2:3], s[20:21]
-; SI-NEXT: v_fma_f64 v[2:3], v[2:3], s[22:23], v[22:23]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], s[10:11], v[16:17]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[12:13]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[18:19]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[24:25]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[26:27]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[42:43]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[44:45]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[46:47]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[56:57]
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], 1.0
-; SI-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], 1.0
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], s[10:11], v[16:17]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[10:11], s[28:29], v[4:5]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[12:13]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[12:13], s[40:41], v[4:5]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[18:19]
-; SI-NEXT: v_cvt_i32_f64_e32 v4, v[12:13]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[24:25]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[26:27]
-; SI-NEXT: v_ldexp_f64 v[4:5], v[22:23], v4
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[42:43]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[44:45]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[46:47]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[56:57]
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], 1.0
-; SI-NEXT: v_fma_f64 v[16:17], v[2:3], v[0:1], 1.0
-; SI-NEXT: v_cvt_i32_f64_e32 v0, v[8:9]
-; SI-NEXT: v_cvt_i32_f64_e32 v2, v[10:11]
-; SI-NEXT: v_mov_b32_e32 v10, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e64 v5, v10, v5, s[10:11]
-; SI-NEXT: v_ldexp_f64 v[0:1], v[18:19], v0
-; SI-NEXT: v_cndmask_b32_e64 v5, 0, v5, s[12:13]
-; SI-NEXT: v_cndmask_b32_e64 v3, v10, v1, s[4:5]
-; SI-NEXT: v_ldexp_f64 v[1:2], v[20:21], v2
-; SI-NEXT: s_and_b64 s[4:5], s[8:9], s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v8, s[8:9]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[14:15]
-; SI-NEXT: s_and_b64 vcc, s[12:13], s[10:11]
-; SI-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[6:7]
-; SI-NEXT: v_ldexp_f64 v[8:9], v[16:17], v8
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[40:41], v[6:7]
-; SI-NEXT: v_cndmask_b32_e32 v6, v10, v9, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[14:15]
-; SI-NEXT: v_cndmask_b32_e64 v7, 0, v6, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_v4f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[8:9], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s8, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v16, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v15, s8, v16, v9
+; SI-SDAG-NEXT: v_mov_b32_e32 v14, 0
+; SI-SDAG-NEXT: v_add_f64 v[10:11], v[8:9], v[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[10:11], v[10:11], -v[14:15]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[8:9]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
+; SI-SDAG-NEXT: v_mul_f64 v[10:11], v[2:3], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s17, 0xbfd34413
+; SI-SDAG-NEXT: v_bfi_b32 v15, s8, v16, v11
+; SI-SDAG-NEXT: v_add_f64 v[12:13], v[10:11], v[14:15]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[10:11]|, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[12:13], v[12:13], -v[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s14, 0xa994fd21
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v11, v13, v11, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v10, v12, v10, vcc
+; SI-SDAG-NEXT: v_mul_f64 v[12:13], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x3c49dc1d
+; SI-SDAG-NEXT: v_bfi_b32 v15, s8, v16, v13
+; SI-SDAG-NEXT: v_add_f64 v[17:18], v[12:13], v[14:15]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[12:13]|, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[17:18], v[17:18], -v[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x494ea3e9
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v13, v18, v13, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v17, v12, vcc
+; SI-SDAG-NEXT: v_mul_f64 v[17:18], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s21, 0xbcaf48ad
+; SI-SDAG-NEXT: v_bfi_b32 v15, s8, v16, v18
+; SI-SDAG-NEXT: v_add_f64 v[19:20], v[17:18], v[14:15]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[17:18]|, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[19:20], -v[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0xbbb55516
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v14, v14, v17, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[8:9], s[16:17], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v15, v15, v18, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[8:9], s[14:15], v[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x40026bb1
+; SI-SDAG-NEXT: v_mul_f64 v[18:19], v[16:17], s[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfca7ab0c
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[16:17], s[22:23], v[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s10, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s11, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v17, s5
+; SI-SDAG-NEXT: v_mov_b32_e32 v16, s4
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], s[10:11], v[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s13, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[12:13]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s26, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s27, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[26:27]
+; SI-SDAG-NEXT: s_mov_b32 s42, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s43, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[42:43]
+; SI-SDAG-NEXT: s_mov_b32 s44, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s45, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[44:45]
+; SI-SDAG-NEXT: s_mov_b32 s46, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s47, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[46:47]
+; SI-SDAG-NEXT: s_mov_b32 s56, 11
+; SI-SDAG-NEXT: s_mov_b32 s57, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[56:57]
+; SI-SDAG-NEXT: s_mov_b32 s28, 0
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s29, 0x40900000
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[18:19], v[20:21], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[10:11], s[16:17], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s40, 0
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[10:11], s[14:15], v[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s41, 0xc090cc00
+; SI-SDAG-NEXT: v_mul_f64 v[22:23], v[20:21], s[20:21]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[28:29], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[20:21], s[22:23], v[22:23]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[40:41], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], s[10:11], v[16:17]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[6:7], s[28:29], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[12:13]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[8:9], s[40:41], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[46:47]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[20:21], v[0:1], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[20:21], v[0:1], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[12:13], s[16:17], v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[12:13], s[14:15], v[0:1]
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[0:1], s[22:23], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[14:15], s[16:17], v[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[14:15], s[14:15], v[2:3]
+; SI-SDAG-NEXT: s_and_b64 s[14:15], vcc, s[4:5]
+; SI-SDAG-NEXT: v_mul_f64 v[22:23], v[2:3], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[2:3], s[22:23], v[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], s[10:11], v[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[46:47]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[22:23], v[0:1], v[22:23], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], s[10:11], v[16:17]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[10:11], s[28:29], v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[12:13]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[12:13], s[40:41], v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[18:19]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v4, v[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[26:27]
+; SI-SDAG-NEXT: v_ldexp_f64 v[4:5], v[22:23], v4
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[46:47]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[2:3], v[0:1], 1.0
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v0, v[8:9]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v2, v[10:11]
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, v10, v5, s[10:11]
+; SI-SDAG-NEXT: v_ldexp_f64 v[0:1], v[18:19], v0
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v5, s[12:13]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, v10, v1, s[4:5]
+; SI-SDAG-NEXT: v_ldexp_f64 v[1:2], v[20:21], v2
+; SI-SDAG-NEXT: s_and_b64 s[4:5], s[8:9], s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v8, s[8:9]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[14:15]
+; SI-SDAG-NEXT: s_and_b64 vcc, s[12:13], s[10:11]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[28:29], v[6:7]
+; SI-SDAG-NEXT: v_ldexp_f64 v[8:9], v[16:17], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[40:41], v[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v6, v10, v9, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[14:15]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v6, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_v4f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[8:9], v[0:1], v[16:17]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v10, 0x80000000, v9
+; SI-GISEL-NEXT: v_or_b32_e32 v13, 0x43300000, v10
+; SI-GISEL-NEXT: v_add_f64 v[14:15], v[8:9], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[13:14], v[14:15], -v[12:13]
+; SI-GISEL-NEXT: v_mul_f64 v[18:19], v[2:3], v[16:17]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[8:9]|, v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0xbbb55516
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, v13, v8, vcc
+; SI-GISEL-NEXT: v_and_b32_e32 v13, 0x80000000, v19
+; SI-GISEL-NEXT: v_or_b32_e32 v13, 0x43300000, v13
+; SI-GISEL-NEXT: v_add_f64 v[20:21], v[18:19], v[12:13]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v14, v9, vcc
+; SI-GISEL-NEXT: v_add_f64 v[14:15], v[20:21], -v[12:13]
+; SI-GISEL-NEXT: v_mul_f64 v[20:21], v[4:5], v[16:17]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[18:19]|, v[10:11]
+; SI-GISEL-NEXT: v_and_b32_e32 v13, 0x80000000, v21
+; SI-GISEL-NEXT: v_or_b32_e32 v13, 0x43300000, v13
+; SI-GISEL-NEXT: v_add_f64 v[22:23], v[20:21], v[12:13]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v14, v14, v18, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v15, v15, v19, vcc
+; SI-GISEL-NEXT: v_add_f64 v[18:19], v[22:23], -v[12:13]
+; SI-GISEL-NEXT: v_mul_f64 v[22:23], v[6:7], v[16:17]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[20:21]|, v[10:11]
+; SI-GISEL-NEXT: v_and_b32_e32 v13, 0x80000000, v23
+; SI-GISEL-NEXT: v_or_b32_e32 v13, 0x43300000, v13
+; SI-GISEL-NEXT: v_add_f64 v[24:25], v[22:23], v[12:13]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v16, v18, v20, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v17, v19, v21, vcc
+; SI-GISEL-NEXT: v_add_f64 v[12:13], v[24:25], -v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v18, 0x509f79ff
+; SI-GISEL-NEXT: v_mov_b32_e32 v19, 0x3fd34413
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[22:23]|, v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], -v[8:9], v[18:19], v[0:1]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v10, v12, v22, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v24, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v25, 0xbc49dc1d
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v11, v13, v23, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], -v[14:15], v[18:19], v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], -v[8:9], v[24:25], v[20:21]
+; SI-GISEL-NEXT: v_mov_b32_e32 v20, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v21, 0xbcaf48ad
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], -v[14:15], v[24:25], v[22:23]
+; SI-GISEL-NEXT: v_mul_f64 v[26:27], v[12:13], v[20:21]
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x40026bb1
+; SI-GISEL-NEXT: v_mul_f64 v[30:31], v[22:23], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[12:13], v[28:29], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[22:23], v[28:29], v[30:31]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[26:27], -v[16:17], v[18:19], v[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], -v[10:11], v[18:19], v[6:7]
+; SI-GISEL-NEXT: v_fma_f64 v[34:35], v[12:13], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[26:27], -v[16:17], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], -v[10:11], v[24:25], v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[34:35], v[36:37]
+; SI-GISEL-NEXT: v_mov_b32_e32 v34, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v35, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[34:35]
+; SI-GISEL-NEXT: v_mul_f64 v[38:39], v[26:27], v[20:21]
+; SI-GISEL-NEXT: v_mov_b32_e32 v48, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v49, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[26:27], v[26:27], v[28:29], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v38, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v39, 0x3f56c16c
+; SI-GISEL-NEXT: v_mul_f64 v[20:21], v[18:19], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v50, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v51, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[18:19], v[28:29], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[12:13], v[24:25], v[50:51]
+; SI-GISEL-NEXT: v_mov_b32_e32 v24, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v25, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[12:13], v[20:21], v[24:25]
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[12:13], v[20:21], v[28:29]
+; SI-GISEL-NEXT: v_mov_b32_e32 v52, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v53, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[12:13], v[20:21], v[52:53]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[12:13], v[20:21], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v10, v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[12:13], v[20:21], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[26:27], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[12:13], v8
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[24:25]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], v[52:53]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[22:23], v[12:13], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[18:19], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v32, v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[20:21], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[22:23], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, 0
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], v[50:51]
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, 0x40900000
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], v[24:25]
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], v[52:53]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v33, 0, v8, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[16:17]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[24:25]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[26:27], v[14:15], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0x7ff00000
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[28:29]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v17, v16, v9, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[14:15], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[4:5], v[30:31]
+; SI-GISEL-NEXT: v_mov_b32_e32 v22, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v23, 0xc090cc00
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], v[52:53]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v15, v16, v9, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[22:23]
+; SI-GISEL-NEXT: v_ldexp_f64 v[12:13], v[12:13], v32
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 s[4:5], v[2:3], v[30:31]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v33, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v17, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[2:3], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[18:19], v[20:21], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v13, v16, v13, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v12, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v13, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[4:5], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[18:19], v[8:9], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v14, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v15, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[8:9], v10
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[6:7], v[30:31]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[6:7], v[22:23]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_v4f64:
; VI-SDAG: ; %bb.0:
@@ -5063,80 +5545,154 @@ define amdgpu_ps <8 x i32> @s_exp10_v4f64(<4 x double> inreg %in) #0 {
}
define double @v_exp10_fabs_f64(double %in) #0 {
-; SI-LABEL: v_exp10_fabs_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_fabs_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_fabs_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], |v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], |v[0:1]|
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_fabs_f64:
; VI-SDAG: ; %bb.0:
@@ -5403,80 +5959,154 @@ define double @v_exp10_fabs_f64(double %in) #0 {
}
define double @v_exp10_fneg_fabs_f64(double %in) #0 {
-; SI-LABEL: v_exp10_fneg_fabs_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0xc00a934f
-; SI-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -|v[0:1]|
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc0900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x4090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_fneg_fabs_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc00a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -|v[0:1]|
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc0900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x4090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_fneg_fabs_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], -|v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], -|v[0:1]|
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, -|v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, -|v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_fneg_fabs_f64:
; VI-SDAG: ; %bb.0:
@@ -5744,80 +6374,154 @@ define double @v_exp10_fneg_fabs_f64(double %in) #0 {
}
define double @v_exp10_fneg_f64(double %in) #0 {
-; SI-LABEL: v_exp10_fneg_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0xc00a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc0900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x4090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_fneg_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc00a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], -v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc0900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x4090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_fneg_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], -v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], -v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, -v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, -v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_fneg_f64:
; VI-SDAG: ; %bb.0:
@@ -6084,74 +6788,142 @@ define double @v_exp10_fneg_f64(double %in) #0 {
}
define double @v_exp10_f64_fast(double %in) #0 {
-; SI-LABEL: v_exp10_f64_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_fast:
; VI-SDAG: ; %bb.0:
@@ -6393,80 +7165,154 @@ define double @v_exp10_f64_fast(double %in) #0 {
}
define double @v_exp10_f64_afn(double %in) #0 {
-; SI-LABEL: v_exp10_f64_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_afn:
; VI-SDAG: ; %bb.0:
@@ -6732,74 +7578,142 @@ define double @v_exp10_f64_afn(double %in) #0 {
}
define double @v_exp10_f64_ninf(double %in) #0 {
-; SI-LABEL: v_exp10_f64_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_ninf:
; VI-SDAG: ; %bb.0:
@@ -7041,80 +7955,154 @@ define double @v_exp10_f64_ninf(double %in) #0 {
}
define double @v_exp10_f64_nnan(double %in) #0 {
-; SI-LABEL: v_exp10_f64_nnan:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_nnan:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_nnan:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_nnan:
; VI-SDAG: ; %bb.0:
@@ -7380,80 +8368,154 @@ define double @v_exp10_f64_nnan(double %in) #0 {
}
define double @v_fabs_exp10_f64_afn(double %in) #0 {
-; SI-LABEL: v_fabs_exp10_f64_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_fabs_exp10_f64_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], |v[0:1]|
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_fabs_exp10_f64_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], |v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], |v[0:1]|
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_fabs_exp10_f64_afn:
; VI-SDAG: ; %bb.0:
@@ -7720,74 +8782,142 @@ define double @v_fabs_exp10_f64_afn(double %in) #0 {
}
define double @v_exp10_f64_nnan_ninf(double %in) #0 {
-; SI-LABEL: v_exp10_f64_nnan_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_nnan_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_nnan_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_nnan_ninf:
; VI-SDAG: ; %bb.0:
@@ -8271,81 +9401,156 @@ define double @v_exp10_f64_from_fpext_f16(half %src) #0 {
}
define double @v_exp10_f64_from_fpext_f32(float %src) #0 {
-; SI-LABEL: v_exp10_f64_from_fpext_f32:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_from_fpext_f32:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_from_fpext_f32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[8:9], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x509f79ff
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], -v[2:3], v[10:11], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[2:3], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_from_fpext_f32:
; VI-SDAG: ; %bb.0:
@@ -8839,80 +10044,154 @@ define double @v_exp10_f64_from_fpext_math_f16(half %src0, half %src1) #0 {
}
define double @v_exp10_f64_contract(double %in) #0 {
-; SI-LABEL: v_exp10_f64_contract:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_contract:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_contract:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_contract:
; VI-SDAG: ; %bb.0:
@@ -9178,74 +10457,142 @@ define double @v_exp10_f64_contract(double %in) #0 {
}
define double @v_exp10_f64_contract_nnan_ninf(double %in) #0 {
-; SI-LABEL: v_exp10_f64_contract_nnan_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, 0x979a371
-; SI-NEXT: s_mov_b32 s5, 0x400a934f
-; SI-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v4, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s4, v4, v3
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x509f79ff
-; SI-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s5, 0xbfd34413
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: s_mov_b32 s4, 0xa994fd21
-; SI-NEXT: s_mov_b32 s5, 0x3c49dc1d
-; SI-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x494ea3e9
-; SI-NEXT: s_mov_b32 s5, 0xbcaf48ad
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xbbb55516
-; SI-NEXT: s_mov_b32 s5, 0x40026bb1
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f64_contract_nnan_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x979a371
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x400a934f
+; SI-SDAG-NEXT: v_mul_f64 v[2:3], v[0:1], s[4:5]
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s4, v4, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x509f79ff
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbfd34413
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[0:1]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xa994fd21
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c49dc1d
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[2:3], s[4:5], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x494ea3e9
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xbcaf48ad
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xbbb55516
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40026bb1
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f64_contract_nnan_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x979a371
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x400a934f
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[6:7], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x509f79ff
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fd34413
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[2:3], v[6:7], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x494ea3e9
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbcaf48ad
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xbbb55516
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x40026bb1
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp10_f64_contract_nnan_ninf:
; VI-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll
index f8b14292c346c..39f30786c0850 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll
@@ -7,72 +7,138 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX900,GCN-GISEL,GFX900-GISEL %s
define double @v_exp2_f64(double %in) #0 {
-; SI-LABEL: v_exp2_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64:
; VI-SDAG: ; %bb.0:
@@ -306,102 +372,197 @@ define double @v_exp2_f64(double %in) #0 {
}
define <2 x double> @v_exp2_v2f64(<2 x double> %in) #0 {
-; SI-LABEL: v_exp2_v2f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s24, -2
-; SI-NEXT: v_mov_b32_e32 v14, 0x43300000
-; SI-NEXT: v_bfi_b32 v5, s24, v14, v1
-; SI-NEXT: v_mov_b32_e32 v4, 0
-; SI-NEXT: v_add_f64 v[6:7], v[0:1], v[4:5]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[5:6], v[6:7], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s6, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v7, v6, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
-; SI-NEXT: v_bfi_b32 v5, s24, v14, v3
-; SI-NEXT: v_add_f64 v[14:15], v[2:3], v[4:5]
-; SI-NEXT: v_add_f64 v[8:9], v[0:1], -v[6:7]
-; SI-NEXT: v_add_f64 v[4:5], v[14:15], -v[4:5]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
-; SI-NEXT: s_mov_b32 s7, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[10:11], v[8:9], s[6:7]
-; SI-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; SI-NEXT: s_mov_b32 s8, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s9, 0x3fe62e42
-; SI-NEXT: v_add_f64 v[14:15], v[2:3], -v[4:5]
-; SI-NEXT: v_fma_f64 v[8:9], v[8:9], s[8:9], v[10:11]
-; SI-NEXT: s_mov_b32 s10, 0xfca7ab0c
-; SI-NEXT: s_mov_b32 s11, 0x3e928af3
-; SI-NEXT: s_mov_b32 s12, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s13, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v10, s10
-; SI-NEXT: v_mov_b32_e32 v11, s11
-; SI-NEXT: v_mul_f64 v[16:17], v[14:15], s[6:7]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], s[12:13], v[10:11]
-; SI-NEXT: s_mov_b32 s10, 0x623fde64
-; SI-NEXT: s_mov_b32 s11, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[14:15], v[14:15], s[8:9], v[16:17]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[10:11]
-; SI-NEXT: s_mov_b32 s14, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s15, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], s[12:13], v[10:11]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[14:15]
-; SI-NEXT: s_mov_b32 s16, 0x14761f6e
-; SI-NEXT: s_mov_b32 s17, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[10:11]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[16:17]
-; SI-NEXT: s_mov_b32 s18, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s19, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[14:15]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[18:19]
-; SI-NEXT: s_mov_b32 s20, 0x11122322
-; SI-NEXT: s_mov_b32 s21, 0x3f811111
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[16:17]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[20:21]
-; SI-NEXT: s_mov_b32 s22, 0x555502a1
-; SI-NEXT: s_mov_b32 s23, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[18:19]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[22:23]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[20:21]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[4:5]
-; SI-NEXT: s_mov_b32 s6, 11
-; SI-NEXT: s_mov_b32 s7, 0x3fe00000
-; SI-NEXT: v_cvt_i32_f64_e32 v16, v[6:7]
-; SI-NEXT: v_fma_f64 v[6:7], v[14:15], v[10:11], s[22:23]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[6:7]
-; SI-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], s[4:5]
-; SI-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], 1.0
-; SI-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], s[6:7]
-; SI-NEXT: v_fma_f64 v[8:9], v[8:9], v[12:13], 1.0
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x40900000
-; SI-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s8, 0
-; SI-NEXT: s_mov_b32 s9, 0xc090cc00
-; SI-NEXT: v_cvt_i32_f64_e32 v4, v[4:5]
-; SI-NEXT: v_ldexp_f64 v[8:9], v[8:9], v16
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[0:1]
-; SI-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], 1.0
-; SI-NEXT: v_mov_b32_e32 v10, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v5, v10, v9, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v5, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v8, vcc
-; SI-NEXT: v_ldexp_f64 v[4:5], v[6:7], v4
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[2:3]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v5, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_v2f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s24, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v14, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v5, s24, v14, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[0:1], v[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[5:6], v[6:7], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v6, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
+; SI-SDAG-NEXT: v_bfi_b32 v5, s24, v14, v3
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[2:3], v[4:5]
+; SI-SDAG-NEXT: v_add_f64 v[8:9], v[0:1], -v[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[14:15], -v[4:5]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[10:11], v[8:9], s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
+; SI-SDAG-NEXT: s_mov_b32 s8, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s9, 0x3fe62e42
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[2:3], -v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[8:9], s[8:9], v[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s10, 0xfca7ab0c
+; SI-SDAG-NEXT: s_mov_b32 s11, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s12, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s13, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SI-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SI-SDAG-NEXT: v_mul_f64 v[16:17], v[14:15], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], s[12:13], v[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s10, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s11, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[14:15], s[8:9], v[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s14, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], s[12:13], v[10:11]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s17, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[10:11]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[14:15]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[22:23]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[14:15], v[10:11], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s6, 11
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3fe00000
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v16, v[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[14:15], v[10:11], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[8:9], v[12:13], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x40900000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s8, 0
+; SI-SDAG-NEXT: s_mov_b32 s9, 0xc090cc00
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v4, v[4:5]
+; SI-SDAG-NEXT: v_ldexp_f64 v[8:9], v[8:9], v16
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], 1.0
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v5, v10, v9, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v5, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v8, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[4:5], v[6:7], v4
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[2:3]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[8:9], v[2:3]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v5, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_v2f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v7, 0x43300000, v4
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -v[6:7]
+; SI-GISEL-NEXT: v_and_b32_e32 v7, 0x80000000, v3
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[8:9]
+; SI-GISEL-NEXT: v_or_b32_e32 v7, 0x43300000, v7
+; SI-GISEL-NEXT: v_add_f64 v[10:11], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[10:11], -v[6:7]
+; SI-GISEL-NEXT: v_add_f64 v[10:11], v[0:1], -v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fe62e42
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v7, v3, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[10:11], v[14:15], v[12:13]
+; SI-GISEL-NEXT: v_add_f64 v[12:13], v[2:3], -v[6:7]
+; SI-GISEL-NEXT: v_mul_f64 v[8:9], v[12:13], v[8:9]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[12:13], v[14:15], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[8:9], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[16:17], v[10:11], v[16:17], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[10:11], v[16:17], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x40900000
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0xc090cc00
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[14:15]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], v[0:1], v[16:17]
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[8:9], v[12:13], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[8:9], v[0:1], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v0, v[4:5]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v4, v[6:7]
+; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[10:11], v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x7ff00000
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[4:5], v[8:9], v4
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[2:3], v[14:15]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], v[2:3], v[16:17]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v10, v5, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_v2f64:
; VI-SDAG: ; %bb.0:
@@ -731,130 +892,254 @@ define <2 x double> @v_exp2_v2f64(<2 x double> %in) #0 {
}
define <3 x double> @v_exp2_v3f64(<3 x double> %in) #0 {
-; SI-LABEL: v_exp2_v3f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s46, -2
-; SI-NEXT: v_mov_b32_e32 v10, 0x43300000
-; SI-NEXT: v_bfi_b32 v7, s46, v10, v1
-; SI-NEXT: v_mov_b32_e32 v6, 0
-; SI-NEXT: v_add_f64 v[8:9], v[0:1], v[6:7]
-; SI-NEXT: s_mov_b32 s8, -1
-; SI-NEXT: s_mov_b32 s9, 0x432fffff
-; SI-NEXT: v_add_f64 v[7:8], v[8:9], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[8:9]
-; SI-NEXT: s_mov_b32 s10, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v12, v8, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v11, v7, v0, vcc
-; SI-NEXT: v_add_f64 v[7:8], v[0:1], -v[11:12]
-; SI-NEXT: s_mov_b32 s11, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[13:14], v[7:8], s[10:11]
-; SI-NEXT: s_mov_b32 s12, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s13, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[13:14], v[7:8], s[12:13], v[13:14]
-; SI-NEXT: s_mov_b32 s4, 0xfca7ab0c
-; SI-NEXT: s_mov_b32 s5, 0x3e928af3
-; SI-NEXT: s_mov_b32 s14, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s15, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], s[14:15], v[8:9]
-; SI-NEXT: s_mov_b32 s16, 0x623fde64
-; SI-NEXT: s_mov_b32 s17, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[16:17]
-; SI-NEXT: s_mov_b32 s18, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s19, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[18:19]
-; SI-NEXT: s_mov_b32 s20, 0x14761f6e
-; SI-NEXT: s_mov_b32 s21, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[20:21]
-; SI-NEXT: s_mov_b32 s22, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s23, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[22:23]
-; SI-NEXT: s_mov_b32 s24, 0x11122322
-; SI-NEXT: s_mov_b32 s25, 0x3f811111
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[24:25]
-; SI-NEXT: s_mov_b32 s26, 0x555502a1
-; SI-NEXT: s_mov_b32 s27, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[26:27]
-; SI-NEXT: s_mov_b32 s28, 0x55555511
-; SI-NEXT: s_mov_b32 s29, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[28:29]
-; SI-NEXT: s_mov_b32 s44, 11
-; SI-NEXT: s_mov_b32 s45, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[44:45]
-; SI-NEXT: v_cvt_i32_f64_e32 v7, v[11:12]
-; SI-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], 1.0
-; SI-NEXT: s_mov_b32 s40, 0
-; SI-NEXT: v_fma_f64 v[11:12], v[13:14], v[15:16], 1.0
-; SI-NEXT: s_mov_b32 s41, 0x40900000
-; SI-NEXT: v_ldexp_f64 v[11:12], v[11:12], v7
-; SI-NEXT: v_bfi_b32 v7, s46, v10, v3
-; SI-NEXT: v_add_f64 v[13:14], v[2:3], v[6:7]
-; SI-NEXT: s_mov_b32 s42, 0
-; SI-NEXT: s_mov_b32 s43, 0xc090cc00
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[42:43], v[0:1]
-; SI-NEXT: v_add_f64 v[0:1], v[13:14], -v[6:7]
-; SI-NEXT: v_bfi_b32 v7, s46, v10, v5
-; SI-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[2:3]|, s[8:9]
-; SI-NEXT: v_add_f64 v[14:15], v[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v20, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v21, v20, v12, vcc
-; SI-NEXT: v_cndmask_b32_e64 v13, v1, v3, s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v12, v0, v2, s[6:7]
-; SI-NEXT: v_add_f64 v[6:7], v[14:15], -v[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[4:5]|, s[8:9]
-; SI-NEXT: v_add_f64 v[0:1], v[2:3], -v[12:13]
-; SI-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[6:7]
-; SI-NEXT: v_mul_f64 v[16:17], v[0:1], s[10:11]
-; SI-NEXT: v_add_f64 v[14:15], v[4:5], -v[6:7]
-; SI-NEXT: v_fma_f64 v[16:17], v[0:1], s[12:13], v[16:17]
-; SI-NEXT: v_mul_f64 v[0:1], v[14:15], s[10:11]
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_fma_f64 v[14:15], v[14:15], s[12:13], v[0:1]
-; SI-NEXT: v_fma_f64 v[0:1], v[16:17], s[14:15], v[8:9]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], s[14:15], v[8:9]
-; SI-NEXT: v_fma_f64 v[18:19], v[16:17], v[0:1], s[16:17]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v11, vcc
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[18:19], s[18:19]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[16:17]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[20:21]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[18:19]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[22:23]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[20:21]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[24:25]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[22:23]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[26:27]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[24:25]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[28:29]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[26:27]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[44:45]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[28:29]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], 1.0
-; SI-NEXT: v_cvt_i32_f64_e32 v12, v[12:13]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[44:45]
-; SI-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], 1.0
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], 1.0
-; SI-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v21, s[4:5]
-; SI-NEXT: v_ldexp_f64 v[10:11], v[10:11], v12
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[2:3]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[42:43], v[2:3]
-; SI-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], 1.0
-; SI-NEXT: v_cndmask_b32_e32 v7, v20, v11, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v7, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
-; SI-NEXT: v_ldexp_f64 v[6:7], v[8:9], v6
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[42:43], v[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v7, v20, v7, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v5, 0, v7, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_v3f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s46, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v7, s46, v10, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; SI-SDAG-NEXT: v_add_f64 v[8:9], v[0:1], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s8, -1
+; SI-SDAG-NEXT: s_mov_b32 s9, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[7:8], v[8:9], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s10, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v12, v8, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v11, v7, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[7:8], v[0:1], -v[11:12]
+; SI-SDAG-NEXT: s_mov_b32 s11, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[13:14], v[7:8], s[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s12, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s13, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[13:14], v[7:8], s[12:13], v[13:14]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfca7ab0c
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s14, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v9, s5
+; SI-SDAG-NEXT: v_mov_b32_e32 v8, s4
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], s[14:15], v[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s16, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s17, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[16:17]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[20:21]
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[22:23]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s26, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s27, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[26:27]
+; SI-SDAG-NEXT: s_mov_b32 s28, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s29, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[28:29]
+; SI-SDAG-NEXT: s_mov_b32 s44, 11
+; SI-SDAG-NEXT: s_mov_b32 s45, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], s[44:45]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v7, v[11:12]
+; SI-SDAG-NEXT: v_fma_f64 v[15:16], v[13:14], v[15:16], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s40, 0
+; SI-SDAG-NEXT: v_fma_f64 v[11:12], v[13:14], v[15:16], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s41, 0x40900000
+; SI-SDAG-NEXT: v_ldexp_f64 v[11:12], v[11:12], v7
+; SI-SDAG-NEXT: v_bfi_b32 v7, s46, v10, v3
+; SI-SDAG-NEXT: v_add_f64 v[13:14], v[2:3], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s42, 0
+; SI-SDAG-NEXT: s_mov_b32 s43, 0xc090cc00
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[42:43], v[0:1]
+; SI-SDAG-NEXT: v_add_f64 v[0:1], v[13:14], -v[6:7]
+; SI-SDAG-NEXT: v_bfi_b32 v7, s46, v10, v5
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[2:3]|, s[8:9]
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v20, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v21, v20, v12, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v13, v1, v3, s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v12, v0, v2, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[6:7], v[14:15], -v[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[4:5]|, s[8:9]
+; SI-SDAG-NEXT: v_add_f64 v[0:1], v[2:3], -v[12:13]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[6:7]
+; SI-SDAG-NEXT: v_mul_f64 v[16:17], v[0:1], s[10:11]
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[4:5], -v[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[16:17], v[0:1], s[12:13], v[16:17]
+; SI-SDAG-NEXT: v_mul_f64 v[0:1], v[14:15], s[10:11]
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[14:15], s[12:13], v[0:1]
+; SI-SDAG-NEXT: v_fma_f64 v[0:1], v[16:17], s[14:15], v[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], s[14:15], v[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[16:17], v[0:1], s[16:17]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v11, vcc
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[18:19], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[18:19]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[20:21]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[22:23]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], 1.0
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v12, v[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[10:11], v[16:17], v[10:11], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], 1.0
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v6, v[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v21, s[4:5]
+; SI-SDAG-NEXT: v_ldexp_f64 v[10:11], v[10:11], v12
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[2:3]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[42:43], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[8:9], v[14:15], v[8:9], 1.0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v20, v11, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v7, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
+; SI-SDAG-NEXT: v_ldexp_f64 v[6:7], v[8:9], v6
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[40:41], v[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[42:43], v[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v7, v20, v7, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v7, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_v3f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v6, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v6
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[0:1], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[6:7], -v[8:9]
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[10:11]
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_add_f64 v[18:19], v[2:3], v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v7, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[12:13], v[0:1], -v[6:7]
+; SI-GISEL-NEXT: v_add_f64 v[18:19], v[18:19], -v[8:9]
+; SI-GISEL-NEXT: v_and_b32_e32 v9, 0x80000000, v5
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3c7abc9e
+; SI-GISEL-NEXT: v_or_b32_e32 v9, 0x43300000, v9
+; SI-GISEL-NEXT: v_mul_f64 v[16:17], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[10:11]
+; SI-GISEL-NEXT: v_add_f64 v[28:29], v[4:5], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v20, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v21, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[12:13], v[20:21], v[16:17]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v17, v19, v3, vcc
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[28:29], -v[8:9]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, v[10:11]
+; SI-GISEL-NEXT: v_add_f64 v[32:33], v[2:3], -v[16:17]
+; SI-GISEL-NEXT: v_mov_b32_e32 v18, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v19, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v22, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v23, 0x3e928af3
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v4, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v5, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[18:19], v[22:23]
+; SI-GISEL-NEXT: v_mul_f64 v[36:37], v[32:33], v[14:15]
+; SI-GISEL-NEXT: v_add_f64 v[50:51], v[4:5], -v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v26, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v27, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[32:33], v[20:21], v[36:37]
+; SI-GISEL-NEXT: v_mul_f64 v[14:15], v[50:51], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[50:51], v[20:21], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[50:51], v[32:33], v[18:19], v[22:23]
+; SI-GISEL-NEXT: v_mov_b32_e32 v34, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v35, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[32:33], v[50:51], v[26:27]
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[32:33], v[22:23], v[30:31]
+; SI-GISEL-NEXT: v_mov_b32_e32 v38, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v39, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[12:13], v[24:25], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[32:33], v[22:23], v[34:35]
+; SI-GISEL-NEXT: v_mov_b32_e32 v24, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v25, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[12:13], v[10:11], v[24:25]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[32:33], v[22:23], v[28:29]
+; SI-GISEL-NEXT: v_mov_b32_e32 v48, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v49, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[12:13], v[10:11], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[30:31]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v26, v[6:7]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[32:33], v[22:23], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[12:13], v[10:11], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[32:33], v[6:7], v[24:25]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[12:13], v[10:11], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[32:33], v[6:7], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[10:11], v[12:13], v[10:11], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[32:33], v[6:7], v[36:37]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[10:11], v[10:11], v26
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[24:25]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[32:33], v[6:7], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v16, v[16:17]
+; SI-GISEL-NEXT: v_mov_b32_e32 v20, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v21, 0xc090cc00
+; SI-GISEL-NEXT: v_mov_b32_e32 v17, 0x7ff00000
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[32:33], v[6:7], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v11, v17, v11, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], v[36:37]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v10, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[6:7], v[6:7], v16
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v11, vcc
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[2:3], v[12:13]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v11, v17, v7, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[14:15], v[18:19], 1.0
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[2:3], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[14:15], v[6:7], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v10, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[6:7], v[6:7], v8
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v11, vcc
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[4:5], v[12:13]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, v17, v7, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[4:5], v[20:21]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v7, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_v3f64:
; VI-SDAG: ; %bb.0:
@@ -1276,158 +1561,311 @@ define <3 x double> @v_exp2_v3f64(<3 x double> %in) #0 {
}
define <4 x double> @v_exp2_v4f64(<4 x double> %in) #0 {
-; SI-LABEL: v_exp2_v4f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s62, -2
-; SI-NEXT: v_mov_b32_e32 v17, 0x43300000
-; SI-NEXT: v_bfi_b32 v11, s62, v17, v1
-; SI-NEXT: v_mov_b32_e32 v10, 0
-; SI-NEXT: v_add_f64 v[8:9], v[0:1], v[10:11]
-; SI-NEXT: s_mov_b32 s14, -1
-; SI-NEXT: s_mov_b32 s15, 0x432fffff
-; SI-NEXT: v_add_f64 v[8:9], v[8:9], -v[10:11]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[14:15]
-; SI-NEXT: s_mov_b32 s18, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v9, v9, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc
-; SI-NEXT: v_add_f64 v[11:12], v[0:1], -v[8:9]
-; SI-NEXT: s_mov_b32 s19, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[13:14], v[11:12], s[18:19]
-; SI-NEXT: s_mov_b32 s20, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s21, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[14:15], v[11:12], s[20:21], v[13:14]
-; SI-NEXT: s_mov_b32 s4, 0xfca7ab0c
-; SI-NEXT: s_mov_b32 s5, 0x3e928af3
-; SI-NEXT: s_mov_b32 s22, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s23, 0x3e5ade15
-; SI-NEXT: v_mov_b32_e32 v13, s5
-; SI-NEXT: v_mov_b32_e32 v12, s4
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], s[22:23], v[12:13]
-; SI-NEXT: s_mov_b32 s24, 0x623fde64
-; SI-NEXT: s_mov_b32 s25, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[24:25]
-; SI-NEXT: s_mov_b32 s26, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s27, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[26:27]
-; SI-NEXT: s_mov_b32 s28, 0x14761f6e
-; SI-NEXT: s_mov_b32 s29, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[28:29]
-; SI-NEXT: s_mov_b32 s40, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s41, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[40:41]
-; SI-NEXT: s_mov_b32 s42, 0x11122322
-; SI-NEXT: s_mov_b32 s43, 0x3f811111
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[42:43]
-; SI-NEXT: s_mov_b32 s44, 0x555502a1
-; SI-NEXT: s_mov_b32 s45, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[44:45]
-; SI-NEXT: s_mov_b32 s46, 0x55555511
-; SI-NEXT: s_mov_b32 s47, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[46:47]
-; SI-NEXT: s_mov_b32 s56, 11
-; SI-NEXT: s_mov_b32 s57, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[56:57]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
-; SI-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], 1.0
-; SI-NEXT: s_mov_b32 s58, 0
-; SI-NEXT: v_fma_f64 v[14:15], v[14:15], v[18:19], 1.0
-; SI-NEXT: s_mov_b32 s59, 0x40900000
-; SI-NEXT: s_mov_b32 s60, 0
-; SI-NEXT: s_mov_b32 s61, 0xc090cc00
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[58:59], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[60:61], v[0:1]
-; SI-NEXT: v_bfi_b32 v11, s62, v17, v3
-; SI-NEXT: v_ldexp_f64 v[8:9], v[14:15], v8
-; SI-NEXT: v_add_f64 v[14:15], v[2:3], v[10:11]
-; SI-NEXT: s_and_b64 s[6:7], s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[6:7]
-; SI-NEXT: v_add_f64 v[14:15], v[14:15], -v[10:11]
-; SI-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[2:3]|, s[14:15]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[16:17], s[58:59], v[2:3]
-; SI-NEXT: v_cndmask_b32_e64 v15, v15, v3, s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v14, v14, v2, s[6:7]
-; SI-NEXT: v_add_f64 v[18:19], v[2:3], -v[14:15]
-; SI-NEXT: v_cvt_i32_f64_e32 v1, v[14:15]
-; SI-NEXT: v_mul_f64 v[20:21], v[18:19], s[18:19]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[10:11], s[60:61], v[2:3]
-; SI-NEXT: v_fma_f64 v[18:19], v[18:19], s[20:21], v[20:21]
-; SI-NEXT: s_and_b64 s[6:7], s[10:11], s[16:17]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], s[22:23], v[12:13]
-; SI-NEXT: v_bfi_b32 v11, s62, v17, v5
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[24:25]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[8:9], s[58:59], v[4:5]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[26:27]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[28:29]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[40:41]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[42:43]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[44:45]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[46:47]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[56:57]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], 1.0
-; SI-NEXT: v_fma_f64 v[18:19], v[18:19], v[20:21], 1.0
-; SI-NEXT: v_ldexp_f64 v[15:16], v[18:19], v1
-; SI-NEXT: v_cndmask_b32_e64 v2, 0, v15, s[6:7]
-; SI-NEXT: v_add_f64 v[14:15], v[4:5], v[10:11]
-; SI-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[4:5]|, s[14:15]
-; SI-NEXT: v_add_f64 v[14:15], v[14:15], -v[10:11]
-; SI-NEXT: v_bfi_b32 v11, s62, v17, v7
-; SI-NEXT: v_cndmask_b32_e64 v15, v15, v5, s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v14, v14, v4, s[6:7]
-; SI-NEXT: v_add_f64 v[18:19], v[4:5], -v[14:15]
-; SI-NEXT: v_cvt_i32_f64_e32 v1, v[14:15]
-; SI-NEXT: v_mul_f64 v[20:21], v[18:19], s[18:19]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[6:7], s[60:61], v[4:5]
-; SI-NEXT: v_fma_f64 v[18:19], v[18:19], s[20:21], v[20:21]
-; SI-NEXT: s_and_b64 s[12:13], s[6:7], s[8:9]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], s[22:23], v[12:13]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[24:25]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[26:27]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[28:29]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[40:41]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[42:43]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[44:45]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[46:47]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[56:57]
-; SI-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], 1.0
-; SI-NEXT: v_fma_f64 v[18:19], v[18:19], v[20:21], 1.0
-; SI-NEXT: v_ldexp_f64 v[14:15], v[18:19], v1
-; SI-NEXT: v_add_f64 v[17:18], v[6:7], v[10:11]
-; SI-NEXT: v_cndmask_b32_e64 v4, 0, v14, s[12:13]
-; SI-NEXT: v_add_f64 v[10:11], v[17:18], -v[10:11]
-; SI-NEXT: v_cmp_gt_f64_e64 s[12:13], |v[6:7]|, s[14:15]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[14:15], s[58:59], v[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v11, v11, v7, s[12:13]
-; SI-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[12:13]
-; SI-NEXT: v_add_f64 v[17:18], v[6:7], -v[10:11]
-; SI-NEXT: v_cvt_i32_f64_e32 v1, v[10:11]
-; SI-NEXT: v_mul_f64 v[19:20], v[17:18], s[18:19]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[12:13], s[60:61], v[6:7]
-; SI-NEXT: v_fma_f64 v[17:18], v[17:18], s[20:21], v[19:20]
-; SI-NEXT: v_mov_b32_e32 v7, 0x7ff00000
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], s[22:23], v[12:13]
-; SI-NEXT: s_and_b64 s[18:19], s[12:13], s[14:15]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[24:25]
-; SI-NEXT: v_cndmask_b32_e64 v3, v7, v16, s[16:17]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[26:27]
-; SI-NEXT: v_cndmask_b32_e64 v5, v7, v15, s[8:9]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[28:29]
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[10:11]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[40:41]
-; SI-NEXT: v_cndmask_b32_e64 v5, 0, v5, s[6:7]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[42:43]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[44:45]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[46:47]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[56:57]
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], 1.0
-; SI-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], 1.0
-; SI-NEXT: v_ldexp_f64 v[10:11], v[12:13], v1
-; SI-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc
-; SI-NEXT: v_cndmask_b32_e64 v7, v7, v11, s[14:15]
-; SI-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[18:19]
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
-; SI-NEXT: v_cndmask_b32_e64 v7, 0, v7, s[12:13]
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_v4f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s62, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v17, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v11, s62, v17, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v10, 0
+; SI-SDAG-NEXT: v_add_f64 v[8:9], v[0:1], v[10:11]
+; SI-SDAG-NEXT: s_mov_b32 s14, -1
+; SI-SDAG-NEXT: s_mov_b32 s15, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[8:9], v[8:9], -v[10:11]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[14:15]
+; SI-SDAG-NEXT: s_mov_b32 s18, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[11:12], v[0:1], -v[8:9]
+; SI-SDAG-NEXT: s_mov_b32 s19, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[13:14], v[11:12], s[18:19]
+; SI-SDAG-NEXT: s_mov_b32 s20, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s21, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[11:12], s[20:21], v[13:14]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfca7ab0c
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s22, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s23, 0x3e5ade15
+; SI-SDAG-NEXT: v_mov_b32_e32 v13, s5
+; SI-SDAG-NEXT: v_mov_b32_e32 v12, s4
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], s[22:23], v[12:13]
+; SI-SDAG-NEXT: s_mov_b32 s24, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s25, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[24:25]
+; SI-SDAG-NEXT: s_mov_b32 s26, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s27, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[26:27]
+; SI-SDAG-NEXT: s_mov_b32 s28, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s29, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[28:29]
+; SI-SDAG-NEXT: s_mov_b32 s40, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s41, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[40:41]
+; SI-SDAG-NEXT: s_mov_b32 s42, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s43, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[42:43]
+; SI-SDAG-NEXT: s_mov_b32 s44, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s45, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[44:45]
+; SI-SDAG-NEXT: s_mov_b32 s46, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s47, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[46:47]
+; SI-SDAG-NEXT: s_mov_b32 s56, 11
+; SI-SDAG-NEXT: s_mov_b32 s57, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], s[56:57]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[14:15], v[18:19], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s58, 0
+; SI-SDAG-NEXT: v_fma_f64 v[14:15], v[14:15], v[18:19], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s59, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s60, 0
+; SI-SDAG-NEXT: s_mov_b32 s61, 0xc090cc00
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[58:59], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[60:61], v[0:1]
+; SI-SDAG-NEXT: v_bfi_b32 v11, s62, v17, v3
+; SI-SDAG-NEXT: v_ldexp_f64 v[8:9], v[14:15], v8
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[2:3], v[10:11]
+; SI-SDAG-NEXT: s_and_b64 s[6:7], s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[14:15], -v[10:11]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[2:3]|, s[14:15]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[16:17], s[58:59], v[2:3]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v15, v15, v3, s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v14, v14, v2, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[18:19], v[2:3], -v[14:15]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v1, v[14:15]
+; SI-SDAG-NEXT: v_mul_f64 v[20:21], v[18:19], s[18:19]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[10:11], s[60:61], v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[18:19], s[20:21], v[20:21]
+; SI-SDAG-NEXT: s_and_b64 s[6:7], s[10:11], s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], s[22:23], v[12:13]
+; SI-SDAG-NEXT: v_bfi_b32 v11, s62, v17, v5
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[24:25]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[8:9], s[58:59], v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[40:41]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[46:47]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[18:19], v[20:21], 1.0
+; SI-SDAG-NEXT: v_ldexp_f64 v[15:16], v[18:19], v1
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v15, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[4:5], v[10:11]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[4:5]|, s[14:15]
+; SI-SDAG-NEXT: v_add_f64 v[14:15], v[14:15], -v[10:11]
+; SI-SDAG-NEXT: v_bfi_b32 v11, s62, v17, v7
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v15, v15, v5, s[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v14, v14, v4, s[6:7]
+; SI-SDAG-NEXT: v_add_f64 v[18:19], v[4:5], -v[14:15]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v1, v[14:15]
+; SI-SDAG-NEXT: v_mul_f64 v[20:21], v[18:19], s[18:19]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[6:7], s[60:61], v[4:5]
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[18:19], s[20:21], v[20:21]
+; SI-SDAG-NEXT: s_and_b64 s[12:13], s[6:7], s[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], s[22:23], v[12:13]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[24:25]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[26:27]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[28:29]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[40:41]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[46:47]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[20:21], v[18:19], v[20:21], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[18:19], v[18:19], v[20:21], 1.0
+; SI-SDAG-NEXT: v_ldexp_f64 v[14:15], v[18:19], v1
+; SI-SDAG-NEXT: v_add_f64 v[17:18], v[6:7], v[10:11]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, v14, s[12:13]
+; SI-SDAG-NEXT: v_add_f64 v[10:11], v[17:18], -v[10:11]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 s[12:13], |v[6:7]|, s[14:15]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[14:15], s[58:59], v[6:7]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v11, v11, v7, s[12:13]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[12:13]
+; SI-SDAG-NEXT: v_add_f64 v[17:18], v[6:7], -v[10:11]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v1, v[10:11]
+; SI-SDAG-NEXT: v_mul_f64 v[19:20], v[17:18], s[18:19]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[12:13], s[60:61], v[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[17:18], v[17:18], s[20:21], v[19:20]
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x7ff00000
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], s[22:23], v[12:13]
+; SI-SDAG-NEXT: s_and_b64 s[18:19], s[12:13], s[14:15]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[24:25]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v16, s[16:17]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[26:27]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, v7, v15, s[8:9]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[28:29]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[10:11]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[40:41]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v5, s[6:7]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[42:43]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[44:45]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[46:47]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], s[56:57]
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], 1.0
+; SI-SDAG-NEXT: v_fma_f64 v[12:13], v[17:18], v[12:13], 1.0
+; SI-SDAG-NEXT: v_ldexp_f64 v[10:11], v[12:13], v1
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v7, v7, v11, s[14:15]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[18:19]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v7, s[12:13]
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_v4f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v8, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v11, 0x43300000, v8
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[0:1], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v18, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v19, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[8:9], v[8:9], -v[10:11]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[18:19]
+; SI-GISEL-NEXT: v_and_b32_e32 v11, 0x80000000, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[16:17], v[0:1], -v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v14, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v15, 0x3c7abc9e
+; SI-GISEL-NEXT: v_or_b32_e32 v11, 0x43300000, v11
+; SI-GISEL-NEXT: v_mul_f64 v[20:21], v[16:17], v[14:15]
+; SI-GISEL-NEXT: v_add_f64 v[22:23], v[2:3], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[16:17], v[12:13], v[20:21]
+; SI-GISEL-NEXT: v_add_f64 v[16:17], v[22:23], -v[10:11]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[18:19]
+; SI-GISEL-NEXT: v_and_b32_e32 v11, 0x80000000, v5
+; SI-GISEL-NEXT: v_or_b32_e32 v11, 0x43300000, v11
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v20, v16, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v21, v17, v3, vcc
+; SI-GISEL-NEXT: v_add_f64 v[16:17], v[4:5], v[10:11]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[4:5]|, v[18:19]
+; SI-GISEL-NEXT: v_add_f64 v[16:17], v[16:17], -v[10:11]
+; SI-GISEL-NEXT: v_add_f64 v[22:23], v[2:3], -v[20:21]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v4, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v5, vcc
+; SI-GISEL-NEXT: v_add_f64 v[26:27], v[4:5], -v[16:17]
+; SI-GISEL-NEXT: v_mul_f64 v[28:29], v[22:23], v[14:15]
+; SI-GISEL-NEXT: v_mul_f64 v[30:31], v[26:27], v[14:15]
+; SI-GISEL-NEXT: v_and_b32_e32 v11, 0x80000000, v7
+; SI-GISEL-NEXT: v_or_b32_e32 v11, 0x43300000, v11
+; SI-GISEL-NEXT: v_fma_f64 v[28:29], v[22:23], v[12:13], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[22:23], v[26:27], v[12:13], v[30:31]
+; SI-GISEL-NEXT: v_add_f64 v[26:27], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, 0x3e928af3
+; SI-GISEL-NEXT: v_add_f64 v[10:11], v[26:27], -v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[26:27], v[24:25], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_mov_b32_e32 v34, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v35, 0x3ec71dee
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[6:7]|, v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[24:25], v[26:27], v[34:35]
+; SI-GISEL-NEXT: v_mov_b32_e32 v26, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v27, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[24:25], v[18:19], v[26:27]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v6, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v7, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, 0x3f2a01a0
+; SI-GISEL-NEXT: v_add_f64 v[38:39], v[6:7], -v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[24:25], v[18:19], v[36:37]
+; SI-GISEL-NEXT: v_mov_b32_e32 v48, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v49, 0x3f56c16c
+; SI-GISEL-NEXT: v_mul_f64 v[14:15], v[38:39], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[24:25], v[18:19], v[48:49]
+; SI-GISEL-NEXT: v_mov_b32_e32 v50, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v51, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[38:39], v[12:13], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[24:25], v[18:19], v[50:51]
+; SI-GISEL-NEXT: v_mov_b32_e32 v18, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v19, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[24:25], v[14:15], v[18:19]
+; SI-GISEL-NEXT: v_mov_b32_e32 v38, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v39, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[24:25], v[14:15], v[38:39]
+; SI-GISEL-NEXT: v_mov_b32_e32 v52, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v53, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[24:25], v[14:15], v[52:53]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[8:9]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[24:25], v[14:15], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v10, v[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[24:25], v[14:15], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[22:23], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[14:15], v8
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], v[52:53]
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[28:29], v[14:15], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[28:29], v[12:13], v[30:31], v[32:33]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v32, v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[24:25], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[28:29], v[34:35]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[48:49]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, 0
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[50:51]
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, 0x40900000
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[18:19]
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[30:31]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[38:39]
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[48:49]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], v[52:53]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v33, 0, v8, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[24:25], v[12:13], v[24:25], v[50:51]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], 1.0
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[16:17]
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[12:13], v[24:25], v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[20:21], v[22:23], v[20:21], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v16, 0x7ff00000
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[12:13], v[18:19], v[38:39]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v17, v16, v9, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[20:21], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[4:5], v[30:31]
+; SI-GISEL-NEXT: v_mov_b32_e32 v28, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0xc090cc00
+; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[12:13], v[18:19], v[52:53]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v20, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v21, v16, v9, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[28:29]
+; SI-GISEL-NEXT: v_ldexp_f64 v[14:15], v[14:15], v32
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 s[4:5], v[2:3], v[30:31]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v33, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v17, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[2:3], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[12:13], v[18:19], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v14, 0, v14, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v15, v16, v15, s[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v14, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v15, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[4:5], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[8:9], v[12:13], v[8:9], 1.0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v20, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v21, vcc
+; SI-GISEL-NEXT: v_ldexp_f64 v[8:9], v[8:9], v10
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[6:7], v[30:31]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[6:7], v[28:29]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_v4f64:
; VI-SDAG: ; %bb.0:
@@ -4528,73 +4966,140 @@ define amdgpu_ps <8 x i32> @s_exp2_v4f64(<4 x double> inreg %in) #0 {
}
define double @v_exp2_fabs_f64(double %in) #0 {
-; SI-LABEL: v_exp2_fabs_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x43300000
-; SI-NEXT: v_add_f64 v[2:3], |v[0:1]|, s[6:7]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc3300000
-; SI-NEXT: v_add_f64 v[2:3], v[2:3], s[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], |v[0:1]|, -v[2:3]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fabs_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x43300000
+; SI-SDAG-NEXT: v_add_f64 v[2:3], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc3300000
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[2:3], s[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], |v[0:1]|, -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_fabs_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v8, 0x7fffffff, v1
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v8
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], |v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], |v[0:1]|, -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_fabs_f64:
; VI-SDAG: ; %bb.0:
@@ -4829,73 +5334,140 @@ define double @v_exp2_fabs_f64(double %in) #0 {
}
define double @v_exp2_fneg_fabs_f64(double %in) #0 {
-; SI-LABEL: v_exp2_fneg_fabs_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc3300000
-; SI-NEXT: v_add_f64 v[2:3], -|v[0:1]|, s[6:7]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x43300000
-; SI-NEXT: v_add_f64 v[2:3], v[2:3], s[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_or_b32_e32 v4, 0x80000000, v1
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], -|v[0:1]|, -v[2:3]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc0900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x4090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fneg_fabs_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc3300000
+; SI-SDAG-NEXT: v_add_f64 v[2:3], -|v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x43300000
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[2:3], s[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_or_b32_e32 v4, 0x80000000, v1
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], -|v[0:1]|, -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc0900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x4090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_fneg_fabs_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_or_b32_e32 v8, 0x80000000, v1
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v8
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], -|v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], -|v[0:1]|, -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, -|v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, -|v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_fneg_fabs_f64:
; VI-SDAG: ; %bb.0:
@@ -5131,73 +5703,140 @@ define double @v_exp2_fneg_fabs_f64(double %in) #0 {
}
define double @v_exp2_fneg_f64(double %in) #0 {
-; SI-LABEL: v_exp2_fneg_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v6, 0x80000000, v1
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s4, v2, v6
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], -v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], -v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc0900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x4090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fneg_f64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_xor_b32_e32 v6, 0x80000000, v1
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s4, v2, v6
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], -v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], -v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc0900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x4090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_fneg_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_xor_b32_e32 v8, 0x80000000, v1
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v8
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], -v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], -v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, -v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, -v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_fneg_f64:
; VI-SDAG: ; %bb.0:
@@ -5432,66 +6071,126 @@ define double @v_exp2_fneg_f64(double %in) #0 {
}
define double @v_exp2_f64_fast(double %in) #0 {
-; SI-LABEL: v_exp2_f64_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64_fast:
; VI-SDAG: ; %bb.0:
@@ -5701,76 +6400,142 @@ define double @v_exp2_f64_fast(double %in) #0 {
}
define double @v_exp2_f64_afn(double %in) #0 {
-; SI-LABEL: v_exp2_f64_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-SDAG-LABEL: v_exp2_f64_afn:
-; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-LABEL: v_exp2_f64_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp2_f64_afn:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-SDAG-NEXT: v_rndne_f64_e32 v[2:3], v[0:1]
; VI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
; VI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
@@ -6000,66 +6765,126 @@ define double @v_exp2_f64_afn(double %in) #0 {
}
define double @v_exp2_f64_ninf(double %in) #0 {
-; SI-LABEL: v_exp2_f64_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64_ninf:
; VI-SDAG: ; %bb.0:
@@ -6269,72 +7094,138 @@ define double @v_exp2_f64_ninf(double %in) #0 {
}
define double @v_exp2_f64_nnan(double %in) #0 {
-; SI-LABEL: v_exp2_f64_nnan:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64_nnan:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64_nnan:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64_nnan:
; VI-SDAG: ; %bb.0:
@@ -6568,73 +7459,140 @@ define double @v_exp2_f64_nnan(double %in) #0 {
}
define double @v_fabs_exp2_f64_afn(double %in) #0 {
-; SI-LABEL: v_fabs_exp2_f64_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0x43300000
-; SI-NEXT: v_add_f64 v[2:3], |v[0:1]|, s[6:7]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc3300000
-; SI-NEXT: v_add_f64 v[2:3], v[2:3], s[6:7]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], |v[0:1]|, -v[2:3]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_fabs_exp2_f64_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x43300000
+; SI-SDAG-NEXT: v_add_f64 v[2:3], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc3300000
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[2:3], s[6:7]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], |v[0:1]|, -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, s[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_fabs_exp2_f64_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v8, 0x7fffffff, v1
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v8
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], |v[0:1]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], |v[0:1]|, -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_fabs_exp2_f64_afn:
; VI-SDAG: ; %bb.0:
@@ -6830,105 +7788,165 @@ define double @v_fabs_exp2_f64_afn(double %in) #0 {
; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
-; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
-; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
-; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
-; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
-; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 11
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
-; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
-; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; GFX900-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], 1.0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v6, 0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v7, 0xc090cc00
-; GFX900-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, v[6:7]
-; GFX900-GISEL-NEXT: v_ldexp_f64 v[2:3], v[4:5], v2
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
-; GFX900-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x7ff00000
-; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v2, s[4:5]
-; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
-; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
- %fabs = call double @llvm.fabs.f64(double %in)
- %result = call afn double @llvm.exp2.f64(double %fabs)
- ret double %result
-}
-
-define double @v_exp2_f64_nnan_ninf(double %in) #0 {
-; SI-LABEL: v_exp2_f64_nnan_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; GFX900-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; GFX900-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], 1.0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v7, 0xc090cc00
+; GFX900-GISEL-NEXT: v_cmp_nlt_f64_e64 s[4:5], |v[0:1]|, v[6:7]
+; GFX900-GISEL-NEXT: v_ldexp_f64 v[2:3], v[4:5], v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; GFX900-GISEL-NEXT: v_cmp_ngt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x7ff00000
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, v2, s[4:5]
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %fabs = call double @llvm.fabs.f64(double %in)
+ %result = call afn double @llvm.exp2.f64(double %fabs)
+ ret double %result
+}
+
+define double @v_exp2_f64_nnan_ninf(double %in) #0 {
+; SI-SDAG-LABEL: v_exp2_f64_nnan_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64_nnan_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64_nnan_ninf:
; VI-SDAG: ; %bb.0:
@@ -7356,73 +8374,140 @@ define double @v_exp2_f64_from_fpext_f16(half %src) #0 {
}
define double @v_exp2_f64_from_fpext_f32(float %src) #0 {
-; SI-LABEL: v_exp2_f64_from_fpext_f32:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f64_f32_e32 v[1:2], v0
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: v_mov_b32_e32 v3, 0x43300000
-; SI-NEXT: v_bfi_b32 v4, s4, v3, v0
-; SI-NEXT: v_mov_b32_e32 v3, 0
-; SI-NEXT: v_add_f64 v[5:6], v[1:2], v[3:4]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[3:4], v[5:6], -v[3:4]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[1:2]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_add_f64 v[5:6], v[1:2], -v[3:4]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[7:8], v[5:6], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[5:6], v[5:6], s[4:5], v[7:8]
-; SI-NEXT: v_mov_b32_e32 v7, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v8, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], s[4:5], v[7:8]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v0, v[3:4]
-; SI-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[3:4], v[5:6], v[7:8], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[3:4], v[3:4], v0
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[1:2]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[1:2]
-; SI-NEXT: v_mov_b32_e32 v0, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64_from_fpext_f32:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f64_f32_e32 v[1:2], v0
+; SI-SDAG-NEXT: s_brev_b32 s4, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v3, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v4, s4, v3, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SI-SDAG-NEXT: v_add_f64 v[5:6], v[1:2], v[3:4]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[3:4], v[5:6], -v[3:4]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[1:2]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_add_f64 v[5:6], v[1:2], -v[3:4]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[7:8], v[5:6], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[5:6], v[5:6], s[4:5], v[7:8]
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v8, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], s[4:5], v[7:8]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v0, v[3:4]
+; SI-SDAG-NEXT: v_fma_f64 v[7:8], v[5:6], v[7:8], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[3:4], v[5:6], v[7:8], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[3:4], v[3:4], v0
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[1:2]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[1:2]
+; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64_from_fpext_f32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x432fffff
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[6:7], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[6:7], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[8:9], v[4:5], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[10:11], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v10, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[2:3], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64_from_fpext_f32:
; VI-SDAG: ; %bb.0:
@@ -7860,72 +8945,138 @@ define double @v_exp2_f64_from_fpext_math_f16(half %src0, half %src1) #0 {
}
define double @v_exp2_f64_contract(double %in) #0 {
-; SI-LABEL: v_exp2_f64_contract:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0x40900000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
-; SI-NEXT: v_mov_b32_e32 v4, 0x7ff00000
-; SI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64_contract:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x40900000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[6:7], v[0:1]
+; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7ff00000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; SI-SDAG-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v3, s[4:5]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64_contract:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x7ff00000
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64_contract:
; VI-SDAG: ; %bb.0:
@@ -8159,66 +9310,126 @@ define double @v_exp2_f64_contract(double %in) #0 {
}
define double @v_exp2_f64_contract_nnan_ninf(double %in) #0 {
-; SI-LABEL: v_exp2_f64_contract_nnan_ninf:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s6, -2
-; SI-NEXT: v_mov_b32_e32 v2, 0x43300000
-; SI-NEXT: v_bfi_b32 v3, s6, v2, v1
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; SI-NEXT: s_mov_b32 s4, -1
-; SI-NEXT: s_mov_b32 s5, 0x432fffff
-; SI-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; SI-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x3b39803f
-; SI-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; SI-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; SI-NEXT: s_mov_b32 s5, 0x3c7abc9e
-; SI-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0xfefa39ef
-; SI-NEXT: s_mov_b32 s5, 0x3fe62e42
-; SI-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
-; SI-NEXT: v_mov_b32_e32 v7, 0x3e928af3
-; SI-NEXT: s_mov_b32 s4, 0x6a5dcb37
-; SI-NEXT: s_mov_b32 s5, 0x3e5ade15
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
-; SI-NEXT: s_mov_b32 s4, 0x623fde64
-; SI-NEXT: s_mov_b32 s5, 0x3ec71dee
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x7c89e6b0
-; SI-NEXT: s_mov_b32 s5, 0x3efa0199
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x14761f6e
-; SI-NEXT: s_mov_b32 s5, 0x3f2a01a0
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x1852b7b0
-; SI-NEXT: s_mov_b32 s5, 0x3f56c16c
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x11122322
-; SI-NEXT: s_mov_b32 s5, 0x3f811111
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x555502a1
-; SI-NEXT: s_mov_b32 s5, 0x3fa55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 0x55555511
-; SI-NEXT: s_mov_b32 s5, 0x3fc55555
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: s_mov_b32 s4, 11
-; SI-NEXT: s_mov_b32 s5, 0x3fe00000
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
-; SI-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
-; SI-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s4, 0
-; SI-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
-; SI-NEXT: s_mov_b32 s5, 0xc090cc00
-; SI-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
-; SI-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f64_contract_nnan_ninf:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: s_brev_b32 s6, -2
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x43300000
+; SI-SDAG-NEXT: v_bfi_b32 v3, s6, v2, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s4, -1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x432fffff
+; SI-SDAG-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-SDAG-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3b39803f
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3c7abc9e
+; SI-SDAG-NEXT: v_mul_f64 v[6:7], v[4:5], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xfefa39ef
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe62e42
+; SI-SDAG-NEXT: v_fma_f64 v[4:5], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0xfca7ab0c
+; SI-SDAG-NEXT: v_mov_b32_e32 v7, 0x3e928af3
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x6a5dcb37
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3e5ade15
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], s[4:5], v[6:7]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x623fde64
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3ec71dee
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x7c89e6b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3efa0199
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x14761f6e
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f2a01a0
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x1852b7b0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f56c16c
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x11122322
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3f811111
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x555502a1
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fa55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x55555511
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fc55555
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: s_mov_b32 s4, 11
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3fe00000
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], s[4:5]
+; SI-SDAG-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-SDAG-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0
+; SI-SDAG-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0xc090cc00
+; SI-SDAG-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-SDAG-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f64_contract_nnan_ninf:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_or_b32_e32 v3, 0x43300000, v3
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[4:5], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[8:9], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x6a5dcb37
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3e5ade15
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfca7ab0c
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3e928af3
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x623fde64
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3ec71dee
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x7c89e6b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3efa0199
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x14761f6e
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f2a01a0
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x1852b7b0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f56c16c
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x11122322
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3f811111
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x555502a1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fa55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x55555511
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fc55555
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 11
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe00000
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[8:9]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[6:7], 1.0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0xc090cc00
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v8
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_exp2_f64_contract_nnan_ninf:
; VI-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index 46374d439c621..5d9afd1a4a4f0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -1177,66 +1177,105 @@ define amdgpu_gs void @s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls(<2 x
; GFX11-GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v2.h, s1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v2.l, s0
-; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.h, s3
-; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.l, s2
+; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.l, s1
+; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v4.l, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v2.l, s2
+; GFX11-GISEL-NEXT: v_mov_b16_e32 v6.l, v3.l
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
-; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v4.h, s3
-; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v4.l, s2
+; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.l, s3
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s3, v4
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v5
+; GFX11-GISEL-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v2.l, s2
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s1, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s2, v5
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s1, s2, s3
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s3, v3
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
-; GFX11-GISEL-NEXT: v_pk_add_f16 v2, v2, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_add_f16 v2, v4, v2
+; GFX11-GISEL-NEXT: v_pk_add_f16 v2, s0, s1
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s0, s2, s3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: v_pk_add_f16 v2, s0, v2
; GFX11-GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-GISEL-NEXT: s_endpgm
;
; GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
; GISEL: ; %bb.0:
; GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GISEL-NEXT: v_mov_b32_e32 v3, s2
-; GISEL-NEXT: v_mov_b32_e32 v4, s1
+; GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-NEXT: v_mov_b32_e32 v4, s2
; GISEL-NEXT: v_mov_b32_e32 v5, s3
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v3
-; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v4
; GISEL-NEXT: v_cvt_f16_f32_e32 v7, v5
+; GISEL-NEXT: v_readfirstlane_b32 s0, v2
+; GISEL-NEXT: v_readfirstlane_b32 s1, v3
+; GISEL-NEXT: v_readfirstlane_b32 s2, v6
+; GISEL-NEXT: v_readfirstlane_b32 s3, v7
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
-; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v2
-; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v5
-; GISEL-NEXT: v_lshl_or_b32 v5, v7, 16, v6
-; GISEL-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v4
+; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v5
+; GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GISEL-NEXT: s_pack_ll_b32_b16 s1, s2, s3
+; GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GISEL-NEXT: v_readfirstlane_b32 s3, v3
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
-; GISEL-NEXT: v_pk_add_f16 v2, v2, v5
-; GISEL-NEXT: v_pk_add_f16 v2, v3, v2
+; GISEL-NEXT: v_pk_add_f16 v2, s0, s1
+; GISEL-NEXT: s_pack_ll_b32_b16 s0, s2, s3
+; GISEL-NEXT: v_pk_add_f16 v2, s0, v2
; GISEL-NEXT: global_store_dword v[0:1], v2, off
; GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
-; GFX12-NEXT: s_cvt_f16_f32 s1, s1
-; GFX12-NEXT: s_cvt_f16_f32 s0, s0
-; GFX12-NEXT: s_cvt_f16_f32 s4, s3
-; GFX12-NEXT: s_cvt_f16_f32 s5, s2
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 2
-; GFX12-NEXT: s_cvt_f16_f32 s3, s3
-; GFX12-NEXT: s_cvt_f16_f32 s2, s2
-; GFX12-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX12-NEXT: s_pack_ll_b32_b16 s1, s5, s4
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
-; GFX12-NEXT: v_pk_add_f16 v2, s0, s1
-; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_add_f16 v2, s0, v2
-; GFX12-NEXT: global_store_b32 v[0:1], v2, off
-; GFX12-NEXT: s_endpgm
+; GFX12-SDAG-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s1, s1
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s4, s3
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s5, s2
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 2
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s3, s3
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s2, s2
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s1, s5, s4
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
+; GFX12-SDAG-NEXT: v_pk_add_f16 v2, s0, s1
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s0, s2, s3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_pk_add_f16 v2, s0, v2
+; GFX12-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s1, s1
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s4, s2
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s5, s3
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 2
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s2, s2
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s3, s3
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
+; GFX12-GISEL-NEXT: s_add_f16 s0, s0, s4
+; GFX12-GISEL-NEXT: s_add_f16 s1, s1, s5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT: s_add_f16 s0, s2, s0
+; GFX12-GISEL-NEXT: s_add_f16 s1, s3, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-GISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-GISEL-NEXT: s_endpgm
%res1 = call <2 x half> @llvm.fptrunc.round.v2f16.v2f32(<2 x float> %a, metadata !"round.upward")
%res2 = call <2 x half> @llvm.fptrunc.round.v2f16.v2f32(<2 x float> %b, metadata !"round.upward")
%res3 = call <2 x half> @llvm.fptrunc.round.v2f16.v2f32(<2 x float> %b, metadata !"round.downward")
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll b/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll
index 17cc74ee62aab..f6d09aaaec497 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll
@@ -33,10 +33,10 @@ define i32 @func_rounding() {
; GFX678-GISEL-LABEL: func_rounding:
; GFX678-GISEL: ; %bb.0:
; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-GISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 4)
-; GFX678-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX678-GISEL-NEXT: s_getreg_b32 s6, hwreg(HW_REG_MODE, 0, 4)
; GFX678-GISEL-NEXT: s_mov_b32 s4, 0xeb24da71
; GFX678-GISEL-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX678-GISEL-NEXT: s_lshl_b32 s6, s6, 2
; GFX678-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
; GFX678-GISEL-NEXT: s_and_b32 s4, s4, 15
; GFX678-GISEL-NEXT: s_add_i32 s5, s4, 4
@@ -63,10 +63,10 @@ define i32 @func_rounding() {
; GFX9-GISEL-LABEL: func_rounding:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 4)
-; GFX9-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX9-GISEL-NEXT: s_getreg_b32 s6, hwreg(HW_REG_MODE, 0, 4)
; GFX9-GISEL-NEXT: s_mov_b32 s4, 0xeb24da71
; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s6, 2
; GFX9-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
; GFX9-GISEL-NEXT: s_and_b32 s4, s4, 15
; GFX9-GISEL-NEXT: s_add_i32 s5, s4, 4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
index 2a5d6ebfea29d..92adf4ff3d915 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
@@ -280,35 +280,65 @@ define <2 x float> @test_modf_v2f32_only_use_integer(<2 x float> %x) {
}
define { double, double } @test_modf_f64(double %x) {
-; GFX9-LABEL: test_modf_f64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX9-NEXT: s_movk_i32 s4, 0x204
-; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s4
-; GFX9-NEXT: s_brev_b32 s6, -2
-; GFX9-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v5, 0, s[4:5]
-; GFX9-NEXT: v_bfi_b32 v1, s6, v4, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_f64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x204
+; GFX9-SDAG-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s4
+; GFX9-SDAG-NEXT: s_brev_b32 s6, -2
+; GFX9-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v5, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_bfi_b32 v1, s6, v4, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_f64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0x7ff00000
+; GFX9-GISEL-NEXT: v_cmp_eq_f64_e64 s[4:5], |v[0:1]|, v[4:5]
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v4, -2
+; GFX9-GISEL-NEXT: v_add_f64 v[6:7], v[0:1], -v[2:3]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v5, v7, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v6, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v5, v4, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { double, double } @llvm.modf.f64(double %x)
ret { double, double } %result
}
define double @test_modf_f64_only_use_fract(double %x) {
-; GFX9-LABEL: test_modf_f64_only_use_fract:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX9-NEXT: s_movk_i32 s4, 0x204
-; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s4
-; GFX9-NEXT: s_brev_b32 s6, -2
-; GFX9-NEXT: v_add_f64 v[2:3], v[0:1], -v[2:3]
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, 0, s[4:5]
-; GFX9-NEXT: v_bfi_b32 v1, s6, v2, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_f64_only_use_fract:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x204
+; GFX9-SDAG-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s4
+; GFX9-SDAG-NEXT: s_brev_b32 s6, -2
+; GFX9-SDAG-NEXT: v_add_f64 v[2:3], v[0:1], -v[2:3]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v3, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_bfi_b32 v1, s6, v2, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_f64_only_use_fract:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0x7ff00000
+; GFX9-GISEL-NEXT: v_cmp_eq_f64_e64 s[4:5], |v[0:1]|, v[4:5]
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v4, -2
+; GFX9-GISEL-NEXT: v_add_f64 v[2:3], v[0:1], -v[2:3]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v3, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v2, v4, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { double, double } @llvm.modf.f64(double %x)
%result.0 = extractvalue { double, double } %result, 0
ret double %result.0
@@ -326,47 +356,91 @@ define double @test_modf_f64_only_use_integer(double %x) {
}
define { <2 x double>, <2 x double> } @test_modf_v2f64(<2 x double> %x) {
-; GFX9-LABEL: test_modf_v2f64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
-; GFX9-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
-; GFX9-NEXT: s_movk_i32 s6, 0x204
-; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s6
-; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], s6
-; GFX9-NEXT: s_brev_b32 s8, -2
-; GFX9-NEXT: v_add_f64 v[8:9], v[0:1], -v[4:5]
-; GFX9-NEXT: v_add_f64 v[10:11], v[2:3], -v[6:7]
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v8, v9, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v9, v11, 0, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, 0, s[6:7]
-; GFX9-NEXT: v_bfi_b32 v1, s8, v8, v1
-; GFX9-NEXT: v_bfi_b32 v3, s8, v9, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_v2f64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX9-SDAG-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX9-SDAG-NEXT: s_movk_i32 s6, 0x204
+; GFX9-SDAG-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s6
+; GFX9-SDAG-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], s6
+; GFX9-SDAG-NEXT: s_brev_b32 s8, -2
+; GFX9-SDAG-NEXT: v_add_f64 v[8:9], v[0:1], -v[4:5]
+; GFX9-SDAG-NEXT: v_add_f64 v[10:11], v[2:3], -v[6:7]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v8, v9, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v9, v11, 0, s[6:7]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, 0, s[6:7]
+; GFX9-SDAG-NEXT: v_bfi_b32 v1, s8, v8, v1
+; GFX9-SDAG-NEXT: v_bfi_b32 v3, s8, v9, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_v2f64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v9, 0x7ff00000
+; GFX9-GISEL-NEXT: v_cmp_eq_f64_e64 s[4:5], |v[0:1]|, v[8:9]
+; GFX9-GISEL-NEXT: v_cmp_eq_f64_e64 s[6:7], |v[2:3]|, v[8:9]
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v8, -2
+; GFX9-GISEL-NEXT: v_add_f64 v[10:11], v[0:1], -v[4:5]
+; GFX9-GISEL-NEXT: v_add_f64 v[12:13], v[2:3], -v[6:7]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v10, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v9, v11, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v10, v13, 0, s[6:7]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v12, 0, s[6:7]
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v9, v8, v1
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v10, v8, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x double>, <2 x double> } @llvm.modf.v2f64(<2 x double> %x)
ret { <2 x double>, <2 x double> } %result
}
define <2 x double> @test_modf_v2f64_only_use_fract(<2 x double> %x) {
-; GFX9-LABEL: test_modf_v2f64_only_use_fract:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
-; GFX9-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
-; GFX9-NEXT: s_movk_i32 s6, 0x204
-; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s6
-; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], s6
-; GFX9-NEXT: s_brev_b32 s8, -2
-; GFX9-NEXT: v_add_f64 v[4:5], v[0:1], -v[4:5]
-; GFX9-NEXT: v_add_f64 v[6:7], v[2:3], -v[6:7]
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v5, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, 0, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[6:7]
-; GFX9-NEXT: v_bfi_b32 v1, s8, v4, v1
-; GFX9-NEXT: v_bfi_b32 v3, s8, v5, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_v2f64_only_use_fract:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX9-SDAG-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX9-SDAG-NEXT: s_movk_i32 s6, 0x204
+; GFX9-SDAG-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], s6
+; GFX9-SDAG-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], s6
+; GFX9-SDAG-NEXT: s_brev_b32 s8, -2
+; GFX9-SDAG-NEXT: v_add_f64 v[4:5], v[0:1], -v[4:5]
+; GFX9-SDAG-NEXT: v_add_f64 v[6:7], v[2:3], -v[6:7]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v5, 0, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v7, 0, s[6:7]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[6:7]
+; GFX9-SDAG-NEXT: v_bfi_b32 v1, s8, v4, v1
+; GFX9-SDAG-NEXT: v_bfi_b32 v3, s8, v5, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_v2f64_only_use_fract:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[4:5], v[0:1]
+; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[6:7], v[2:3]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v9, 0x7ff00000
+; GFX9-GISEL-NEXT: v_cmp_eq_f64_e64 s[4:5], |v[0:1]|, v[8:9]
+; GFX9-GISEL-NEXT: v_cmp_eq_f64_e64 s[6:7], |v[2:3]|, v[8:9]
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v8, -2
+; GFX9-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[4:5]
+; GFX9-GISEL-NEXT: v_add_f64 v[6:7], v[2:3], -v[6:7]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x80000000, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v4, v5, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v5, v7, 0, s[6:7]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[6:7]
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v4, v8, v1
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v5, v8, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x double>, <2 x double> } @llvm.modf.v2f64(<2 x double> %x)
%result.0 = extractvalue { <2 x double>, <2 x double> } %result, 0
ret <2 x double> %result.0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
index 82d0b4957cdb3..3880e821bdb4d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
@@ -40,29 +40,53 @@ define amdgpu_gfx void @s_set_rounding(i32 inreg %rounding) {
; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: s_set_rounding:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_add_i32 s34, s4, -4
-; GFX10-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX10-NEXT: s_min_u32 s36, s4, s34
-; GFX10-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX10-NEXT: s_lshl_b32 s36, s36, 2
-; GFX10-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_add_i32 s0, s4, -4
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: s_min_u32 s2, s4, s0
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: s_set_rounding:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_add_i32 s34, s4, -4
+; GFX10-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-SDAG-NEXT: s_min_u32 s36, s4, s34
+; GFX10-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_lshl_b32 s36, s36, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_add_i32 s34, s4, -4
+; GFX10-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_min_u32 s34, s4, s34
+; GFX10-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_add_i32 s0, s4, -4
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: s_min_u32 s2, s4, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_add_i32 s0, s4, -4
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_min_u32 s0, s4, s0
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.set.rounding(i32 %rounding)
ret void
}
@@ -143,66 +167,109 @@ define amdgpu_kernel void @s_set_rounding_kernel(i32 inreg %rounding) {
; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
; GFX9-NEXT: s_endpgm
;
-; GFX10-LABEL: s_set_rounding_kernel:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX10-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX10-NEXT: ;;#ASMSTART
-; GFX10-NEXT: ;;#ASMEND
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_add_i32 s3, s2, -4
-; GFX10-NEXT: s_min_u32 s2, s2, s3
-; GFX10-NEXT: s_lshl_b32 s2, s2, 2
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: s_set_rounding_kernel:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: ;;#ASMSTART
-; GFX11-NEXT: ;;#ASMEND
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_add_i32 s3, s2, -4
-; GFX11-NEXT: s_min_u32 s2, s2, s3
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: s_set_rounding_kernel:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX10-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX10-SDAG-NEXT: ;;#ASMSTART
+; GFX10-SDAG-NEXT: ;;#ASMEND
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_add_i32 s3, s2, -4
+; GFX10-SDAG-NEXT: s_min_u32 s2, s2, s3
+; GFX10-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: s_set_rounding_kernel:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dword s0, s[4:5], 0x24
+; GFX10-GISEL-NEXT: ;;#ASMSTART
+; GFX10-GISEL-NEXT: ;;#ASMEND
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX10-GISEL-NEXT: s_min_u32 s2, s0, s1
+; GFX10-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX10-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: s_set_rounding_kernel:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: ;;#ASMSTART
+; GFX11-SDAG-NEXT: ;;#ASMEND
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_add_i32 s3, s2, -4
+; GFX11-SDAG-NEXT: s_min_u32 s2, s2, s3
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: s_set_rounding_kernel:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX11-GISEL-NEXT: ;;#ASMSTART
+; GFX11-GISEL-NEXT: ;;#ASMEND
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s2, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_endpgm
call void @llvm.set.rounding(i32 %rounding)
call void asm sideeffect "",""()
ret void
}
define void @v_set_rounding(i32 %rounding) {
-; GFX6-LABEL: v_set_rounding:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, -4, v0
-; GFX6-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX6-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX6-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX6-NEXT: v_lshr_b64 v[0:1], s[4:5], v0
-; GFX6-NEXT: v_readfirstlane_b32 s4, v0
-; GFX6-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX7-LABEL: v_set_rounding:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, -4, v0
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX7-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX7-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX7-NEXT: v_lshr_b64 v[0:1], s[4:5], v0
-; GFX7-NEXT: v_readfirstlane_b32 s4, v0
-; GFX7-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_set_rounding:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_add_i32_e32 v1, vcc, -4, v0
+; GFX6-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX6-SDAG-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX6-SDAG-NEXT: v_lshr_b64 v[0:1], s[4:5], v0
+; GFX6-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX6-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: v_set_rounding:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX678-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX678-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX678-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX678-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX678-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX678-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-SDAG-LABEL: v_set_rounding:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, -4, v0
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-SDAG-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX7-SDAG-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX7-SDAG-NEXT: v_lshr_b64 v[0:1], s[4:5], v0
+; GFX7-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: v_set_rounding:
; GFX8-SDAG: ; %bb.0:
@@ -217,145 +284,255 @@ define void @v_set_rounding(i32 %rounding) {
; GFX8-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GISEL-LABEL: v_set_rounding:
-; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_add_u32_e32 v1, vcc, -4, v0
-; GFX8-GISEL-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX8-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX8-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX8-GISEL-NEXT: v_lshrrev_b64 v[0:1], v0, s[4:5]
-; GFX8-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX8-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: v_set_rounding:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, -4, v0
-; GFX9-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX9-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, s[4:5]
-; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_set_rounding:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_nc_u32_e32 v1, -4, v0
-; GFX10-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX10-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], v0, s[4:5]
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_set_rounding:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, -4, v0
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_set_rounding:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_add_u32_e32 v1, -4, v0
+; GFX9-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX9-SDAG-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX9-SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, s[4:5]
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: v_set_rounding:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX9-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_set_rounding:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_add_nc_u32_e32 v1, -4, v0
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX10-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, s[4:5]
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_set_rounding:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX10-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX10-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_set_rounding:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, -4, v0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, s[0:1]
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_set_rounding:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.set.rounding(i32 %rounding)
ret void
}
define void @set_rounding_get_rounding() {
-; GFX678-LABEL: set_rounding_get_rounding:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 4)
-; GFX678-NEXT: s_lshl_b32 s6, s4, 2
-; GFX678-NEXT: s_mov_b32 s4, 0xeb24da71
-; GFX678-NEXT: s_mov_b32 s5, 0xc96f385
-; GFX678-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX678-NEXT: s_and_b32 s4, s4, 15
-; GFX678-NEXT: s_add_i32 s5, s4, 4
-; GFX678-NEXT: s_cmp_lt_u32 s4, 4
-; GFX678-NEXT: s_cselect_b32 s4, s4, s5
-; GFX678-NEXT: s_add_i32 s5, s4, -4
-; GFX678-NEXT: s_min_u32 s4, s4, s5
-; GFX678-NEXT: s_lshl_b32 s6, s4, 2
-; GFX678-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX678-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX678-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: set_rounding_get_rounding:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 4)
-; GFX9-NEXT: s_lshl_b32 s6, s4, 2
-; GFX9-NEXT: s_mov_b32 s4, 0xeb24da71
-; GFX9-NEXT: s_mov_b32 s5, 0xc96f385
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX9-NEXT: s_and_b32 s4, s4, 15
-; GFX9-NEXT: s_add_i32 s5, s4, 4
-; GFX9-NEXT: s_cmp_lt_u32 s4, 4
-; GFX9-NEXT: s_cselect_b32 s4, s4, s5
-; GFX9-NEXT: s_add_i32 s5, s4, -4
-; GFX9-NEXT: s_min_u32 s4, s4, s5
-; GFX9-NEXT: s_lshl_b32 s6, s4, 2
-; GFX9-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX9-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: set_rounding_get_rounding:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_getreg_b32 s6, hwreg(HW_REG_MODE, 0, 4)
-; GFX10-NEXT: s_mov_b32 s4, 0xeb24da71
-; GFX10-NEXT: s_mov_b32 s5, 0xc96f385
-; GFX10-NEXT: s_lshl_b32 s6, s6, 2
-; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX10-NEXT: s_and_b32 s4, s4, 15
-; GFX10-NEXT: s_add_i32 s5, s4, 4
-; GFX10-NEXT: s_cmp_lt_u32 s4, 4
-; GFX10-NEXT: s_cselect_b32 s4, s4, s5
-; GFX10-NEXT: s_add_i32 s5, s4, -4
-; GFX10-NEXT: s_min_u32 s6, s4, s5
-; GFX10-NEXT: s_mov_b32 s4, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s5, 0xb73e62d9
-; GFX10-NEXT: s_lshl_b32 s6, s6, 2
-; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: set_rounding_get_rounding:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_getreg_b32 s2, hwreg(HW_REG_MODE, 0, 4)
-; GFX11-NEXT: s_mov_b32 s0, 0xeb24da71
-; GFX11-NEXT: s_mov_b32 s1, 0xc96f385
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_and_b32 s0, s0, 15
-; GFX11-NEXT: s_add_i32 s1, s0, 4
-; GFX11-NEXT: s_cmp_lt_u32 s0, 4
-; GFX11-NEXT: s_cselect_b32 s0, s0, s1
-; GFX11-NEXT: s_add_i32 s1, s0, -4
-; GFX11-NEXT: s_min_u32 s2, s0, s1
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: set_rounding_get_rounding:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 4)
+; GFX678-SDAG-NEXT: s_lshl_b32 s6, s4, 2
+; GFX678-SDAG-NEXT: s_mov_b32 s4, 0xeb24da71
+; GFX678-SDAG-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX678-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX678-SDAG-NEXT: s_and_b32 s4, s4, 15
+; GFX678-SDAG-NEXT: s_add_i32 s5, s4, 4
+; GFX678-SDAG-NEXT: s_cmp_lt_u32 s4, 4
+; GFX678-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX678-SDAG-NEXT: s_add_i32 s5, s4, -4
+; GFX678-SDAG-NEXT: s_min_u32 s4, s4, s5
+; GFX678-SDAG-NEXT: s_lshl_b32 s6, s4, 2
+; GFX678-SDAG-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX678-SDAG-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX678-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: set_rounding_get_rounding:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: s_getreg_b32 s6, hwreg(HW_REG_MODE, 0, 4)
+; GFX678-GISEL-NEXT: s_mov_b32 s4, 0xeb24da71
+; GFX678-GISEL-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX678-GISEL-NEXT: s_lshl_b32 s6, s6, 2
+; GFX678-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX678-GISEL-NEXT: s_and_b32 s4, s4, 15
+; GFX678-GISEL-NEXT: s_add_i32 s5, s4, 4
+; GFX678-GISEL-NEXT: s_cmp_lt_u32 s4, 4
+; GFX678-GISEL-NEXT: s_cselect_b32 s4, s4, s5
+; GFX678-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX678-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX678-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX678-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX678-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX678-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: set_rounding_get_rounding:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 4)
+; GFX9-SDAG-NEXT: s_lshl_b32 s6, s4, 2
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0xeb24da71
+; GFX9-SDAG-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX9-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX9-SDAG-NEXT: s_and_b32 s4, s4, 15
+; GFX9-SDAG-NEXT: s_add_i32 s5, s4, 4
+; GFX9-SDAG-NEXT: s_cmp_lt_u32 s4, 4
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX9-SDAG-NEXT: s_add_i32 s5, s4, -4
+; GFX9-SDAG-NEXT: s_min_u32 s4, s4, s5
+; GFX9-SDAG-NEXT: s_lshl_b32 s6, s4, 2
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX9-SDAG-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX9-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: set_rounding_get_rounding:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_getreg_b32 s6, hwreg(HW_REG_MODE, 0, 4)
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 0xeb24da71
+; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s6, 2
+; GFX9-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX9-GISEL-NEXT: s_and_b32 s4, s4, 15
+; GFX9-GISEL-NEXT: s_add_i32 s5, s4, 4
+; GFX9-GISEL-NEXT: s_cmp_lt_u32 s4, 4
+; GFX9-GISEL-NEXT: s_cselect_b32 s4, s4, s5
+; GFX9-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX9-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: set_rounding_get_rounding:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_getreg_b32 s6, hwreg(HW_REG_MODE, 0, 4)
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0xeb24da71
+; GFX10-SDAG-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX10-SDAG-NEXT: s_lshl_b32 s6, s6, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX10-SDAG-NEXT: s_and_b32 s4, s4, 15
+; GFX10-SDAG-NEXT: s_add_i32 s5, s4, 4
+; GFX10-SDAG-NEXT: s_cmp_lt_u32 s4, 4
+; GFX10-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX10-SDAG-NEXT: s_add_i32 s5, s4, -4
+; GFX10-SDAG-NEXT: s_min_u32 s6, s4, s5
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX10-SDAG-NEXT: s_lshl_b32 s6, s6, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: set_rounding_get_rounding:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_getreg_b32 s6, hwreg(HW_REG_MODE, 0, 4)
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0xeb24da71
+; GFX10-GISEL-NEXT: s_mov_b32 s5, 0xc96f385
+; GFX10-GISEL-NEXT: s_lshl_b32 s6, s6, 2
+; GFX10-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX10-GISEL-NEXT: s_and_b32 s4, s4, 15
+; GFX10-GISEL-NEXT: s_add_i32 s5, s4, 4
+; GFX10-GISEL-NEXT: s_cmp_lt_u32 s4, 4
+; GFX10-GISEL-NEXT: s_cselect_b32 s4, s4, s5
+; GFX10-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX10-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX10-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: set_rounding_get_rounding:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_getreg_b32 s2, hwreg(HW_REG_MODE, 0, 4)
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_and_b32 s0, s0, 15
+; GFX11-SDAG-NEXT: s_add_i32 s1, s0, 4
+; GFX11-SDAG-NEXT: s_cmp_lt_u32 s0, 4
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-SDAG-NEXT: s_add_i32 s1, s0, -4
+; GFX11-SDAG-NEXT: s_min_u32 s2, s0, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: set_rounding_get_rounding:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_getreg_b32 s2, hwreg(HW_REG_MODE, 0, 4)
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_and_b32 s0, s0, 15
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, 4
+; GFX11-GISEL-NEXT: s_cmp_lt_u32 s0, 4
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%rounding = call i32 @llvm.get.rounding()
call void @llvm.set.rounding(i32 %rounding)
ret void
@@ -921,38 +1098,65 @@ define amdgpu_gfx void @s_set_rounding_i2_zeroext(i2 zeroext inreg %rounding) {
; GFX8-GISEL-LABEL: s_set_rounding_i2_zeroext:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: s_and_b32 s34, 0xffff, s4
+; GFX8-GISEL-NEXT: s_and_b32 s34, s4, 3
; GFX8-GISEL-NEXT: s_lshl_b32 s34, s34, 2
; GFX8-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
; GFX8-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: s_set_rounding_i2_zeroext:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s34, 0xffff, s4
-; GFX9-NEXT: s_lshl_b32 s34, s34, 2
-; GFX9-NEXT: s_lshr_b32 s34, 0xa50f, s34
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_i2_zeroext:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_and_b32 s34, 0xffff, s4
-; GFX10-NEXT: s_lshl_b32 s34, s34, 2
-; GFX10-NEXT: s_lshr_b32 s34, 0xa50f, s34
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_i2_zeroext:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s4
-; GFX11-NEXT: s_lshl_b32 s0, s0, 2
-; GFX11-NEXT: s_lshr_b32 s0, 0xa50f, s0
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: s_set_rounding_i2_zeroext:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_and_b32 s34, 0xffff, s4
+; GFX9-SDAG-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-SDAG-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_i2_zeroext:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_and_b32 s34, s4, 3
+; GFX9-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_i2_zeroext:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_and_b32 s34, 0xffff, s4
+; GFX10-SDAG-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-SDAG-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_i2_zeroext:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_and_b32 s34, s4, 3
+; GFX10-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_i2_zeroext:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_and_b32 s0, 0xffff, s4
+; GFX11-SDAG-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-SDAG-NEXT: s_lshr_b32 s0, 0xa50f, s0
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_i2_zeroext:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_and_b32 s0, s4, 3
+; GFX11-GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-GISEL-NEXT: s_lshr_b32 s0, 0xa50f, s0
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%zext.rounding = zext i2 %rounding to i32
call void @llvm.set.rounding(i32 %zext.rounding)
ret void
@@ -999,7 +1203,7 @@ define amdgpu_gfx void @s_set_rounding_i2_signext(i2 signext inreg %rounding) {
; GFX8-GISEL-LABEL: s_set_rounding_i2_signext:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: s_sext_i32_i16 s34, s4
+; GFX8-GISEL-NEXT: s_bfe_i32 s34, s4, 0x20000
; GFX8-GISEL-NEXT: s_add_i32 s35, s34, -4
; GFX8-GISEL-NEXT: s_min_u32 s34, s34, s35
; GFX8-GISEL-NEXT: s_lshl_b32 s36, s34, 2
@@ -1009,44 +1213,83 @@ define amdgpu_gfx void @s_set_rounding_i2_signext(i2 signext inreg %rounding) {
; GFX8-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: s_set_rounding_i2_signext:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s34, s4
-; GFX9-NEXT: s_add_i32 s35, s34, -4
-; GFX9-NEXT: s_min_u32 s34, s34, s35
-; GFX9-NEXT: s_lshl_b32 s36, s34, 2
-; GFX9-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX9-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_i2_signext:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_sext_i32_i16 s34, s4
-; GFX10-NEXT: s_add_i32 s35, s34, -4
-; GFX10-NEXT: s_min_u32 s36, s34, s35
-; GFX10-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX10-NEXT: s_lshl_b32 s36, s36, 2
-; GFX10-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_i2_signext:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_sext_i32_i16 s0, s4
-; GFX11-NEXT: s_add_i32 s1, s0, -4
-; GFX11-NEXT: s_min_u32 s2, s0, s1
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: s_set_rounding_i2_signext:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_sext_i32_i16 s34, s4
+; GFX9-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX9-SDAG-NEXT: s_min_u32 s34, s34, s35
+; GFX9-SDAG-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_i2_signext:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_bfe_i32 s34, s4, 0x20000
+; GFX9-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX9-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX9-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_i2_signext:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_sext_i32_i16 s34, s4
+; GFX10-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX10-SDAG-NEXT: s_min_u32 s36, s34, s35
+; GFX10-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-SDAG-NEXT: s_lshl_b32 s36, s36, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_i2_signext:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_bfe_i32 s34, s4, 0x20000
+; GFX10-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX10-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX10-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_i2_signext:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_sext_i32_i16 s0, s4
+; GFX11-SDAG-NEXT: s_add_i32 s1, s0, -4
+; GFX11-SDAG-NEXT: s_min_u32 s2, s0, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_i2_signext:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_bfe_i32 s0, s4, 0x20000
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%sext.rounding = sext i2 %rounding to i32
call void @llvm.set.rounding(i32 %sext.rounding)
ret void
@@ -1093,7 +1336,7 @@ define amdgpu_gfx void @s_set_rounding_i3_signext(i3 signext inreg %rounding) {
; GFX8-GISEL-LABEL: s_set_rounding_i3_signext:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: s_sext_i32_i16 s34, s4
+; GFX8-GISEL-NEXT: s_bfe_i32 s34, s4, 0x30000
; GFX8-GISEL-NEXT: s_add_i32 s35, s34, -4
; GFX8-GISEL-NEXT: s_min_u32 s34, s34, s35
; GFX8-GISEL-NEXT: s_lshl_b32 s36, s34, 2
@@ -1103,44 +1346,83 @@ define amdgpu_gfx void @s_set_rounding_i3_signext(i3 signext inreg %rounding) {
; GFX8-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: s_set_rounding_i3_signext:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s34, s4
-; GFX9-NEXT: s_add_i32 s35, s34, -4
-; GFX9-NEXT: s_min_u32 s34, s34, s35
-; GFX9-NEXT: s_lshl_b32 s36, s34, 2
-; GFX9-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX9-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_i3_signext:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_sext_i32_i16 s34, s4
-; GFX10-NEXT: s_add_i32 s35, s34, -4
-; GFX10-NEXT: s_min_u32 s36, s34, s35
-; GFX10-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX10-NEXT: s_lshl_b32 s36, s36, 2
-; GFX10-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_i3_signext:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_sext_i32_i16 s0, s4
-; GFX11-NEXT: s_add_i32 s1, s0, -4
-; GFX11-NEXT: s_min_u32 s2, s0, s1
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: s_set_rounding_i3_signext:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_sext_i32_i16 s34, s4
+; GFX9-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX9-SDAG-NEXT: s_min_u32 s34, s34, s35
+; GFX9-SDAG-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_i3_signext:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_bfe_i32 s34, s4, 0x30000
+; GFX9-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX9-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX9-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_i3_signext:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_sext_i32_i16 s34, s4
+; GFX10-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX10-SDAG-NEXT: s_min_u32 s36, s34, s35
+; GFX10-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-SDAG-NEXT: s_lshl_b32 s36, s36, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_i3_signext:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_bfe_i32 s34, s4, 0x30000
+; GFX10-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX10-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX10-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_i3_signext:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_sext_i32_i16 s0, s4
+; GFX11-SDAG-NEXT: s_add_i32 s1, s0, -4
+; GFX11-SDAG-NEXT: s_min_u32 s2, s0, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_i3_signext:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_bfe_i32 s0, s4, 0x30000
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%sext.rounding = sext i3 %rounding to i32
call void @llvm.set.rounding(i32 %sext.rounding)
ret void
@@ -1187,7 +1469,7 @@ define amdgpu_gfx void @s_set_rounding_i3_zeroext(i3 zeroext inreg %rounding) {
; GFX8-GISEL-LABEL: s_set_rounding_i3_zeroext:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: s_and_b32 s34, 0xffff, s4
+; GFX8-GISEL-NEXT: s_and_b32 s34, s4, 7
; GFX8-GISEL-NEXT: s_add_i32 s35, s34, -4
; GFX8-GISEL-NEXT: s_min_u32 s34, s34, s35
; GFX8-GISEL-NEXT: s_lshl_b32 s36, s34, 2
@@ -1197,44 +1479,83 @@ define amdgpu_gfx void @s_set_rounding_i3_zeroext(i3 zeroext inreg %rounding) {
; GFX8-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: s_set_rounding_i3_zeroext:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s34, 0xffff, s4
-; GFX9-NEXT: s_add_i32 s35, s34, -4
-; GFX9-NEXT: s_min_u32 s34, s34, s35
-; GFX9-NEXT: s_lshl_b32 s36, s34, 2
-; GFX9-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX9-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_i3_zeroext:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_and_b32 s34, 0xffff, s4
-; GFX10-NEXT: s_add_i32 s35, s34, -4
-; GFX10-NEXT: s_min_u32 s36, s34, s35
-; GFX10-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX10-NEXT: s_lshl_b32 s36, s36, 2
-; GFX10-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_i3_zeroext:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s4
-; GFX11-NEXT: s_add_i32 s1, s0, -4
-; GFX11-NEXT: s_min_u32 s2, s0, s1
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: s_set_rounding_i3_zeroext:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_and_b32 s34, 0xffff, s4
+; GFX9-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX9-SDAG-NEXT: s_min_u32 s34, s34, s35
+; GFX9-SDAG-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_i3_zeroext:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_and_b32 s34, s4, 7
+; GFX9-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX9-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX9-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_i3_zeroext:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_and_b32 s34, 0xffff, s4
+; GFX10-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX10-SDAG-NEXT: s_min_u32 s36, s34, s35
+; GFX10-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-SDAG-NEXT: s_lshl_b32 s36, s36, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_i3_zeroext:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_and_b32 s34, s4, 7
+; GFX10-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX10-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX10-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_i3_zeroext:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_and_b32 s0, 0xffff, s4
+; GFX11-SDAG-NEXT: s_add_i32 s1, s0, -4
+; GFX11-SDAG-NEXT: s_min_u32 s2, s0, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_i3_zeroext:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_and_b32 s0, s4, 7
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%sext.rounding = zext i3 %rounding to i32
call void @llvm.set.rounding(i32 %sext.rounding)
ret void
@@ -1287,37 +1608,77 @@ define amdgpu_gfx void @s_set_rounding_select_0_1(i32 inreg %cond) {
}
define amdgpu_gfx void @s_set_rounding_select_1_3(i32 inreg %cond) {
-; GFX678-LABEL: s_set_rounding_select_1_3:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: s_cmp_eq_u32 s4, 0
-; GFX678-NEXT: s_cselect_b32 s34, 0xa50, 10
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: s_set_rounding_select_1_3:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: s_cselect_b32 s34, 0xa50, 10
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_select_1_3:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_cselect_b32 s34, 0xa50, 10
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_select_1_3:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s0, 0xa50, 10
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: s_set_rounding_select_1_3:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-SDAG-NEXT: s_cselect_b32 s34, 0xa50, 10
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: s_set_rounding_select_1_3:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-GISEL-NEXT: s_cselect_b32 s34, 1, 3
+; GFX678-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX678-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: s_set_rounding_select_1_3:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-SDAG-NEXT: s_cselect_b32 s34, 0xa50, 10
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_select_1_3:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s34, 1, 3
+; GFX9-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_select_1_3:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-SDAG-NEXT: s_cselect_b32 s34, 0xa50, 10
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_select_1_3:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s34, 1, 3
+; GFX10-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_select_1_3:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, 0xa50, 10
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_select_1_3:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, 1, 3
+; GFX11-GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-GISEL-NEXT: s_lshr_b32 s0, 0xa50f, s0
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 1, i32 3
call void @llvm.set.rounding(i32 %rounding)
@@ -1325,43 +1686,103 @@ define amdgpu_gfx void @s_set_rounding_select_1_3(i32 inreg %cond) {
}
define void @v_set_rounding_select_1_3(i32 %cond) {
-; GFX678-LABEL: v_set_rounding_select_1_3:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: v_mov_b32_e32 v1, 0xa50
-; GFX678-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX678-NEXT: v_cndmask_b32_e32 v0, 10, v1, vcc
-; GFX678-NEXT: v_readfirstlane_b32 s4, v0
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: v_set_rounding_select_1_3:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xa50
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 10, v1, vcc
-; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_set_rounding_select_1_3:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 10, 0xa50, vcc_lo
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_set_rounding_select_1_3:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 10, 0xa50, vcc_lo
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: v_set_rounding_select_1_3:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: v_mov_b32_e32 v1, 0xa50
+; GFX678-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX678-SDAG-NEXT: v_cndmask_b32_e32 v0, 10, v1, vcc
+; GFX678-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: v_set_rounding_select_1_3:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX678-GISEL-NEXT: v_cndmask_b32_e64 v0, 3, 1, vcc
+; GFX678-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX678-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX678-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX678-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX678-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX678-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX678-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: v_set_rounding_select_1_3:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0xa50
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, 10, v1, vcc
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: v_set_rounding_select_1_3:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 3, 1, vcc
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX9-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX9-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_set_rounding_select_1_3:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, 10, 0xa50, vcc_lo
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_set_rounding_select_1_3:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 3, 1, vcc_lo
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_add_i32 s5, s4, -4
+; GFX10-GISEL-NEXT: s_min_u32 s4, s4, s5
+; GFX10-GISEL-NEXT: s_mov_b32 s5, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s6, s4, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_set_rounding_select_1_3:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, 10, 0xa50, vcc_lo
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_set_rounding_select_1_3:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v0, 3, 1, vcc_lo
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 1, i32 3
call void @llvm.set.rounding(i32 %rounding)
@@ -1369,41 +1790,85 @@ define void @v_set_rounding_select_1_3(i32 %cond) {
}
define amdgpu_gfx void @s_set_rounding_select_2_0(i32 inreg %cond) {
-; GFX678-LABEL: s_set_rounding_select_2_0:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: s_cmp_eq_u32 s4, 0
-; GFX678-NEXT: s_movk_i32 s34, 0xa5
-; GFX678-NEXT: s_cselect_b32 s34, s34, 0xa50f
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: s_set_rounding_select_2_0:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: s_movk_i32 s34, 0xa5
-; GFX9-NEXT: s_cselect_b32 s34, s34, 0xa50f
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_select_2_0:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_movk_i32 s34, 0xa5
-; GFX10-NEXT: s_cselect_b32 s34, s34, 0xa50f
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_select_2_0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: s_movk_i32 s0, 0xa5
-; GFX11-NEXT: s_cselect_b32 s0, s0, 0xa50f
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: s_set_rounding_select_2_0:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-SDAG-NEXT: s_movk_i32 s34, 0xa5
+; GFX678-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa50f
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: s_set_rounding_select_2_0:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX678-GISEL-NEXT: s_lshl_b32 s34, s34, 1
+; GFX678-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX678-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: s_set_rounding_select_2_0:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-SDAG-NEXT: s_movk_i32 s34, 0xa5
+; GFX9-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa50f
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_select_2_0:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX9-GISEL-NEXT: s_lshl_b32 s34, s34, 1
+; GFX9-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_select_2_0:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-SDAG-NEXT: s_movk_i32 s34, 0xa5
+; GFX10-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa50f
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_select_2_0:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX10-GISEL-NEXT: s_lshl_b32 s34, s34, 1
+; GFX10-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_select_2_0:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-SDAG-NEXT: s_movk_i32 s0, 0xa5
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, s0, 0xa50f
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_select_2_0:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-GISEL-NEXT: s_lshr_b32 s0, 0xa50f, s0
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 2, i32 0
call void @llvm.set.rounding(i32 %rounding)
@@ -1411,41 +1876,85 @@ define amdgpu_gfx void @s_set_rounding_select_2_0(i32 inreg %cond) {
}
define amdgpu_gfx void @s_set_rounding_select_2_1(i32 inreg %cond) {
-; GFX678-LABEL: s_set_rounding_select_2_1:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: s_cmp_eq_u32 s4, 0
-; GFX678-NEXT: s_movk_i32 s34, 0xa5
-; GFX678-NEXT: s_cselect_b32 s34, s34, 0xa50
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: s_set_rounding_select_2_1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: s_movk_i32 s34, 0xa5
-; GFX9-NEXT: s_cselect_b32 s34, s34, 0xa50
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_select_2_1:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_movk_i32 s34, 0xa5
-; GFX10-NEXT: s_cselect_b32 s34, s34, 0xa50
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_select_2_1:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: s_movk_i32 s0, 0xa5
-; GFX11-NEXT: s_cselect_b32 s0, s0, 0xa50
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: s_set_rounding_select_2_1:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-SDAG-NEXT: s_movk_i32 s34, 0xa5
+; GFX678-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa50
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: s_set_rounding_select_2_1:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX678-GISEL-NEXT: s_add_i32 s34, s34, 1
+; GFX678-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX678-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: s_set_rounding_select_2_1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-SDAG-NEXT: s_movk_i32 s34, 0xa5
+; GFX9-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa50
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_select_2_1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX9-GISEL-NEXT: s_add_i32 s34, s34, 1
+; GFX9-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_select_2_1:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-SDAG-NEXT: s_movk_i32 s34, 0xa5
+; GFX10-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa50
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_select_2_1:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX10-GISEL-NEXT: s_add_i32 s34, s34, 1
+; GFX10-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_select_2_1:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-SDAG-NEXT: s_movk_i32 s0, 0xa5
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, s0, 0xa50
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_select_2_1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-GISEL-NEXT: s_add_i32 s0, s0, 1
+; GFX11-GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-GISEL-NEXT: s_lshr_b32 s0, 0xa50f, s0
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 2, i32 1
call void @llvm.set.rounding(i32 %rounding)
@@ -1453,41 +1962,101 @@ define amdgpu_gfx void @s_set_rounding_select_2_1(i32 inreg %cond) {
}
define amdgpu_gfx void @s_set_rounding_select_1_2(i32 inreg %cond) {
-; GFX678-LABEL: s_set_rounding_select_1_2:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: s_cmp_eq_u32 s4, 0
-; GFX678-NEXT: s_movk_i32 s34, 0xa50
-; GFX678-NEXT: s_cselect_b32 s34, s34, 0xa5
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: s_set_rounding_select_1_2:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: s_movk_i32 s34, 0xa50
-; GFX9-NEXT: s_cselect_b32 s34, s34, 0xa5
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_select_1_2:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_movk_i32 s34, 0xa50
-; GFX10-NEXT: s_cselect_b32 s34, s34, 0xa5
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_select_1_2:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: s_movk_i32 s0, 0xa50
-; GFX11-NEXT: s_cselect_b32 s0, s0, 0xa5
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: s_set_rounding_select_1_2:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-SDAG-NEXT: s_movk_i32 s34, 0xa50
+; GFX678-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa5
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: s_set_rounding_select_1_2:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-GISEL-NEXT: s_cselect_b32 s34, -1, 0
+; GFX678-GISEL-NEXT: s_add_i32 s34, s34, 2
+; GFX678-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX678-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX678-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX678-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX678-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX678-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: s_set_rounding_select_1_2:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-SDAG-NEXT: s_movk_i32 s34, 0xa50
+; GFX9-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa5
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_select_1_2:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s34, -1, 0
+; GFX9-GISEL-NEXT: s_add_i32 s34, s34, 2
+; GFX9-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX9-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX9-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_select_1_2:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-SDAG-NEXT: s_movk_i32 s34, 0xa50
+; GFX10-SDAG-NEXT: s_cselect_b32 s34, s34, 0xa5
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_select_1_2:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s34, -1, 0
+; GFX10-GISEL-NEXT: s_add_i32 s34, s34, 2
+; GFX10-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX10-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX10-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_select_1_2:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-SDAG-NEXT: s_movk_i32 s0, 0xa50
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, s0, 0xa5
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_select_1_2:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, -1, 0
+; GFX11-GISEL-NEXT: s_add_i32 s0, s0, 2
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 1, i32 2
call void @llvm.set.rounding(i32 %rounding)
@@ -1495,37 +2064,77 @@ define amdgpu_gfx void @s_set_rounding_select_1_2(i32 inreg %cond) {
}
define amdgpu_gfx void @s_set_rounding_select_3_0(i32 inreg %cond) {
-; GFX678-LABEL: s_set_rounding_select_3_0:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: s_cmp_eq_u32 s4, 0
-; GFX678-NEXT: s_cselect_b32 s34, 10, 0xa50f
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: s_set_rounding_select_3_0:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: s_cselect_b32 s34, 10, 0xa50f
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_select_3_0:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_cselect_b32 s34, 10, 0xa50f
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_select_3_0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s0, 10, 0xa50f
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: s_set_rounding_select_3_0:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-SDAG-NEXT: s_cselect_b32 s34, 10, 0xa50f
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: s_set_rounding_select_3_0:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-GISEL-NEXT: s_cselect_b32 s34, 3, 0
+; GFX678-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX678-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: s_set_rounding_select_3_0:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-SDAG-NEXT: s_cselect_b32 s34, 10, 0xa50f
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_select_3_0:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s34, 3, 0
+; GFX9-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_select_3_0:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-SDAG-NEXT: s_cselect_b32 s34, 10, 0xa50f
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_select_3_0:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s34, 3, 0
+; GFX10-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-GISEL-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_select_3_0:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, 10, 0xa50f
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_select_3_0:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, 3, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-GISEL-NEXT: s_lshr_b32 s0, 0xa50f, s0
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 3, i32 0
call void @llvm.set.rounding(i32 %rounding)
@@ -1533,61 +2142,121 @@ define amdgpu_gfx void @s_set_rounding_select_3_0(i32 inreg %cond) {
}
define amdgpu_gfx void @s_set_rounding_select_4_0(i32 inreg %cond) {
-; GFX678-LABEL: s_set_rounding_select_4_0:
-; GFX678: ; %bb.0:
-; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX678-NEXT: s_cmp_eq_u32 s4, 0
-; GFX678-NEXT: s_cselect_b32 s34, 4, 0
-; GFX678-NEXT: s_add_i32 s35, s34, -4
-; GFX678-NEXT: s_min_u32 s34, s34, s35
-; GFX678-NEXT: s_lshl_b32 s36, s34, 2
-; GFX678-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX678-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX678-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX678-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: s_set_rounding_select_4_0:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: s_cselect_b32 s34, 4, 0
-; GFX9-NEXT: s_add_i32 s35, s34, -4
-; GFX9-NEXT: s_min_u32 s34, s34, s35
-; GFX9-NEXT: s_lshl_b32 s36, s34, 2
-; GFX9-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX9-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: s_set_rounding_select_4_0:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_cselect_b32 s34, 4, 0
-; GFX10-NEXT: s_add_i32 s35, s34, -4
-; GFX10-NEXT: s_min_u32 s36, s34, s35
-; GFX10-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX10-NEXT: s_lshl_b32 s36, s36, 2
-; GFX10-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_select_4_0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s0, 4, 0
-; GFX11-NEXT: s_add_i32 s1, s0, -4
-; GFX11-NEXT: s_min_u32 s2, s0, s1
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX678-SDAG-LABEL: s_set_rounding_select_4_0:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-SDAG-NEXT: s_cselect_b32 s34, 4, 0
+; GFX678-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX678-SDAG-NEXT: s_min_u32 s34, s34, s35
+; GFX678-SDAG-NEXT: s_lshl_b32 s36, s34, 2
+; GFX678-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX678-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX678-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX678-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX678-GISEL-LABEL: s_set_rounding_select_4_0:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX678-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX678-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX678-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX678-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX678-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX678-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX678-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX678-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX678-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: s_set_rounding_select_4_0:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-SDAG-NEXT: s_cselect_b32 s34, 4, 0
+; GFX9-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX9-SDAG-NEXT: s_min_u32 s34, s34, s35
+; GFX9-SDAG-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: s_set_rounding_select_4_0:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX9-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX9-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX9-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX9-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX9-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX9-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX9-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: s_set_rounding_select_4_0:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-SDAG-NEXT: s_cselect_b32 s34, 4, 0
+; GFX10-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX10-SDAG-NEXT: s_min_u32 s36, s34, s35
+; GFX10-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-SDAG-NEXT: s_lshl_b32 s36, s36, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_select_4_0:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s34, 1, 0
+; GFX10-GISEL-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX10-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX10-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_select_4_0:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, 4, 0
+; GFX11-SDAG-NEXT: s_add_i32 s1, s0, -4
+; GFX11-SDAG-NEXT: s_min_u32 s2, s0, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_select_4_0:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 4, i32 0
call void @llvm.set.rounding(i32 %rounding)
@@ -1623,33 +2292,61 @@ define amdgpu_gfx void @s_set_rounding_select_3_5(i32 inreg %cond) {
; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: s_set_rounding_select_3_5:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_cselect_b32 s34, 3, 5
-; GFX10-NEXT: s_add_i32 s35, s34, -4
-; GFX10-NEXT: s_min_u32 s36, s34, s35
-; GFX10-NEXT: s_mov_b32 s34, 0x1c84a50f
-; GFX10-NEXT: s_mov_b32 s35, 0xb73e62d9
-; GFX10-NEXT: s_lshl_b32 s36, s36, 2
-; GFX10-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
-; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: s_set_rounding_select_3_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s0, 3, 5
-; GFX11-NEXT: s_add_i32 s1, s0, -4
-; GFX11-NEXT: s_min_u32 s2, s0, s1
-; GFX11-NEXT: s_mov_b32 s0, 0x1c84a50f
-; GFX11-NEXT: s_mov_b32 s1, 0xb73e62d9
-; GFX11-NEXT: s_lshl_b32 s2, s2, 2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: s_set_rounding_select_3_5:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-SDAG-NEXT: s_cselect_b32 s34, 3, 5
+; GFX10-SDAG-NEXT: s_add_i32 s35, s34, -4
+; GFX10-SDAG-NEXT: s_min_u32 s36, s34, s35
+; GFX10-SDAG-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-SDAG-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-SDAG-NEXT: s_lshl_b32 s36, s36, 2
+; GFX10-SDAG-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: s_set_rounding_select_3_5:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-GISEL-NEXT: s_cselect_b32 s34, 3, 5
+; GFX10-GISEL-NEXT: s_add_i32 s35, s34, -4
+; GFX10-GISEL-NEXT: s_min_u32 s34, s34, s35
+; GFX10-GISEL-NEXT: s_mov_b32 s35, 0xb73e62d9
+; GFX10-GISEL-NEXT: s_lshl_b32 s36, s34, 2
+; GFX10-GISEL-NEXT: s_mov_b32 s34, 0x1c84a50f
+; GFX10-GISEL-NEXT: s_lshr_b64 s[34:35], s[34:35], s36
+; GFX10-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: s_set_rounding_select_3_5:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, 3, 5
+; GFX11-SDAG-NEXT: s_add_i32 s1, s0, -4
+; GFX11-SDAG-NEXT: s_min_u32 s2, s0, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-SDAG-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-SDAG-NEXT: s_lshl_b32 s2, s2, 2
+; GFX11-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: s_set_rounding_select_3_5:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s0, 3, 5
+; GFX11-GISEL-NEXT: s_add_i32 s1, s0, -4
+; GFX11-GISEL-NEXT: s_min_u32 s0, s0, s1
+; GFX11-GISEL-NEXT: s_mov_b32 s1, 0xb73e62d9
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s0, 2
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x1c84a50f
+; GFX11-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-GISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
%rounding = select i1 %cmp, i32 3, i32 5
call void @llvm.set.rounding(i32 %rounding)
@@ -1657,45 +2354,85 @@ define amdgpu_gfx void @s_set_rounding_select_3_5(i32 inreg %cond) {
}
define amdgpu_kernel void @get_rounding_after_set_rounding_1() {
-; GFX6-LABEL: get_rounding_after_set_rounding_1:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
-; GFX6-NEXT: s_mov_b32 s1, 0xc96f385
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
-; GFX6-NEXT: s_lshl_b32 s2, s0, 2
-; GFX6-NEXT: s_mov_b32 s0, 0xeb24da71
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: s_and_b32 s0, s0, 15
-; GFX6-NEXT: s_add_i32 s1, s0, 4
-; GFX6-NEXT: s_cmp_lt_u32 s0, 4
-; GFX6-NEXT: s_cselect_b32 s4, s0, s1
-; GFX6-NEXT: s_mov_b64 s[0:1], 0
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: s_endpgm
-;
-; GFX7-LABEL: get_rounding_after_set_rounding_1:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
-; GFX7-NEXT: s_mov_b32 s1, 0xc96f385
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
-; GFX7-NEXT: s_lshl_b32 s2, s0, 2
-; GFX7-NEXT: s_mov_b32 s0, 0xeb24da71
-; GFX7-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX7-NEXT: s_and_b32 s0, s0, 15
-; GFX7-NEXT: s_add_i32 s1, s0, 4
-; GFX7-NEXT: s_cmp_lt_u32 s0, 4
-; GFX7-NEXT: s_cselect_b32 s4, s0, s1
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_endpgm
+; GFX6-SDAG-LABEL: get_rounding_after_set_rounding_1:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX6-SDAG-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-SDAG-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX6-SDAG-NEXT: s_lshl_b32 s2, s0, 2
+; GFX6-SDAG-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX6-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX6-SDAG-NEXT: s_and_b32 s0, s0, 15
+; GFX6-SDAG-NEXT: s_add_i32 s1, s0, 4
+; GFX6-SDAG-NEXT: s_cmp_lt_u32 s0, 4
+; GFX6-SDAG-NEXT: s_cselect_b32 s4, s0, s1
+; GFX6-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX6-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: s_endpgm
+;
+; GFX6-GISEL-LABEL: get_rounding_after_set_rounding_1:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX6-GISEL-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX6-GISEL-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX6-GISEL-NEXT: s_getreg_b32 s2, hwreg(HW_REG_MODE, 0, 4)
+; GFX6-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX6-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX6-GISEL-NEXT: s_and_b32 s0, s0, 15
+; GFX6-GISEL-NEXT: s_add_i32 s1, s0, 4
+; GFX6-GISEL-NEXT: s_cmp_lt_u32 s0, 4
+; GFX6-GISEL-NEXT: s_cselect_b32 s2, s0, s1
+; GFX6-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX6-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: s_endpgm
+;
+; GFX7-SDAG-LABEL: get_rounding_after_set_rounding_1:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX7-SDAG-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX7-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-SDAG-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX7-SDAG-NEXT: s_lshl_b32 s2, s0, 2
+; GFX7-SDAG-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX7-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX7-SDAG-NEXT: s_and_b32 s0, s0, 15
+; GFX7-SDAG-NEXT: s_add_i32 s1, s0, 4
+; GFX7-SDAG-NEXT: s_cmp_lt_u32 s0, 4
+; GFX7-SDAG-NEXT: s_cselect_b32 s4, s0, s1
+; GFX7-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_endpgm
+;
+; GFX7-GISEL-LABEL: get_rounding_after_set_rounding_1:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX7-GISEL-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX7-GISEL-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX7-GISEL-NEXT: s_getreg_b32 s2, hwreg(HW_REG_MODE, 0, 4)
+; GFX7-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX7-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX7-GISEL-NEXT: s_and_b32 s0, s0, 15
+; GFX7-GISEL-NEXT: s_add_i32 s1, s0, 4
+; GFX7-GISEL-NEXT: s_cmp_lt_u32 s0, 4
+; GFX7-GISEL-NEXT: s_cselect_b32 s2, s0, s1
+; GFX7-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX7-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_endpgm
;
; GFX8-SDAG-LABEL: get_rounding_after_set_rounding_1:
; GFX8-SDAG: ; %bb.0:
@@ -1719,40 +2456,59 @@ define amdgpu_kernel void @get_rounding_after_set_rounding_1() {
; GFX8-GISEL-LABEL: get_rounding_after_set_rounding_1:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
-; GFX8-GISEL-NEXT: s_mov_b32 s1, 0xc96f385
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX8-GISEL-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
-; GFX8-GISEL-NEXT: s_lshl_b32 s2, s0, 2
; GFX8-GISEL-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX8-GISEL-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX8-GISEL-NEXT: s_getreg_b32 s2, hwreg(HW_REG_MODE, 0, 4)
+; GFX8-GISEL-NEXT: s_lshl_b32 s2, s2, 2
; GFX8-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
; GFX8-GISEL-NEXT: s_and_b32 s0, s0, 15
; GFX8-GISEL-NEXT: s_add_i32 s1, s0, 4
; GFX8-GISEL-NEXT: s_cmp_lt_u32 s0, 4
; GFX8-GISEL-NEXT: s_cselect_b32 s0, s0, s1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8-GISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: s_endpgm
;
-; GFX9-LABEL: get_rounding_after_set_rounding_1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
-; GFX9-NEXT: s_mov_b32 s1, 0xc96f385
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
-; GFX9-NEXT: s_lshl_b32 s2, s0, 2
-; GFX9-NEXT: s_mov_b32 s0, 0xeb24da71
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: s_and_b32 s0, s0, 15
-; GFX9-NEXT: s_add_i32 s1, s0, 4
-; GFX9-NEXT: s_cmp_lt_u32 s0, 4
-; GFX9-NEXT: s_cselect_b32 s0, s0, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: global_store_dword v[0:1], v2, off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: get_rounding_after_set_rounding_1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX9-SDAG-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX9-SDAG-NEXT: s_lshl_b32 s2, s0, 2
+; GFX9-SDAG-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX9-SDAG-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-SDAG-NEXT: s_and_b32 s0, s0, 15
+; GFX9-SDAG-NEXT: s_add_i32 s1, s0, 4
+; GFX9-SDAG-NEXT: s_cmp_lt_u32 s0, 4
+; GFX9-SDAG-NEXT: s_cselect_b32 s0, s0, s1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: get_rounding_after_set_rounding_1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX9-GISEL-NEXT: s_mov_b32 s0, 0xeb24da71
+; GFX9-GISEL-NEXT: s_mov_b32 s1, 0xc96f385
+; GFX9-GISEL-NEXT: s_getreg_b32 s2, hwreg(HW_REG_MODE, 0, 4)
+; GFX9-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX9-GISEL-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-GISEL-NEXT: s_and_b32 s0, s0, 15
+; GFX9-GISEL-NEXT: s_add_i32 s1, s0, 4
+; GFX9-GISEL-NEXT: s_cmp_lt_u32 s0, 4
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s0, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-LABEL: get_rounding_after_set_rounding_1:
; GFX10: ; %bb.0:
@@ -1795,16 +2551,3 @@ define amdgpu_kernel void @get_rounding_after_set_rounding_1() {
store volatile i32 %set.mode, ptr addrspace(1) null
ret void
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10-GISEL: {{.*}}
-; GFX10-SDAG: {{.*}}
-; GFX11-GISEL: {{.*}}
-; GFX11-SDAG: {{.*}}
-; GFX6-GISEL: {{.*}}
-; GFX6-SDAG: {{.*}}
-; GFX678-GISEL: {{.*}}
-; GFX678-SDAG: {{.*}}
-; GFX7-GISEL: {{.*}}
-; GFX7-SDAG: {{.*}}
-; GFX9-GISEL: {{.*}}
-; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index 5c6eb93a6165f..ab59691705fbd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -52,41 +52,71 @@ define amdgpu_kernel void @sqrt_f16(
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: sqrt_f16:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_sqrt_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
-; GFX11-TRUE16-NEXT: s_endpgm
+; GFX11-TRUE16-SDAG-LABEL: sqrt_f16:
+; GFX11-TRUE16-SDAG: ; %bb.0: ; %entry
+; GFX11-TRUE16-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s10, s6
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s11, s7
+; GFX11-TRUE16-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX11-TRUE16-SDAG-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX11-TRUE16-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-SDAG-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-TRUE16-SDAG-NEXT: s_endpgm
;
-; GFX11-FAKE16-LABEL: sqrt_f16:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-FAKE16-NEXT: s_mov_b32 s11, s7
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_sqrt_f16_e32 v0, v0
-; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-TRUE16-GISEL-LABEL: sqrt_f16:
+; GFX11-TRUE16-GISEL: ; %bb.0: ; %entry
+; GFX11-TRUE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-GISEL-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-TRUE16-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-TRUE16-GISEL-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
+; GFX11-TRUE16-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-GISEL-NEXT: v_sqrt_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-TRUE16-GISEL-NEXT: s_endpgm
+;
+; GFX11-FAKE16-SDAG-LABEL: sqrt_f16:
+; GFX11-FAKE16-SDAG: ; %bb.0: ; %entry
+; GFX11-FAKE16-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s10, s6
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s11, s7
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s8, s2
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s9, s3
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX11-FAKE16-SDAG-NEXT: buffer_load_u16 v0, off, s[8:11], 0
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX11-FAKE16-SDAG-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-FAKE16-SDAG-NEXT: s_endpgm
+;
+; GFX11-FAKE16-GISEL-LABEL: sqrt_f16:
+; GFX11-FAKE16-GISEL: ; %bb.0: ; %entry
+; GFX11-FAKE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-FAKE16-GISEL-NEXT: buffer_load_u16 v0, off, s[4:7], 0
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_sqrt_f16_e32 v0, s2
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-FAKE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-FAKE16-GISEL-NEXT: s_endpgm
;
; GFX12-TRUE16-SDAG-LABEL: sqrt_f16:
; GFX12-TRUE16-SDAG: ; %bb.0: ; %entry
@@ -216,26 +246,45 @@ define amdgpu_kernel void @sqrt_v2f16(
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: sqrt_v2f16:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_sqrt_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sqrt_f16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
-; GFX11-TRUE16-NEXT: s_endpgm
+; GFX11-TRUE16-SDAG-LABEL: sqrt_v2f16:
+; GFX11-TRUE16-SDAG: ; %bb.0: ; %entry
+; GFX11-TRUE16-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s10, s6
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s11, s7
+; GFX11-TRUE16-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX11-TRUE16-SDAG-NEXT: buffer_load_b32 v0, off, s[8:11], 0
+; GFX11-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX11-TRUE16-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.h, v1.l
+; GFX11-TRUE16-SDAG-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-TRUE16-SDAG-NEXT: s_endpgm
+;
+; GFX11-TRUE16-GISEL-LABEL: sqrt_v2f16:
+; GFX11-TRUE16-GISEL: ; %bb.0: ; %entry
+; GFX11-TRUE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-GISEL-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-TRUE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-GISEL-NEXT: v_sqrt_f16_e32 v0.l, s2
+; GFX11-TRUE16-GISEL-NEXT: v_sqrt_f16_e32 v1.l, s3
+; GFX11-TRUE16-GISEL-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-TRUE16-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-TRUE16-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-TRUE16-GISEL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX11-TRUE16-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-TRUE16-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-TRUE16-GISEL-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: sqrt_v2f16:
; GFX11-FAKE16: ; %bb.0: ; %entry
@@ -260,26 +309,44 @@ define amdgpu_kernel void @sqrt_v2f16(
; GFX11-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
; GFX11-FAKE16-NEXT: s_endpgm
;
-; GFX12-TRUE16-LABEL: sqrt_v2f16:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-TRUE16-NEXT: v_sqrt_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sqrt_f16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
-; GFX12-TRUE16-NEXT: s_endpgm
+; GFX12-TRUE16-SDAG-LABEL: sqrt_v2f16:
+; GFX12-TRUE16-SDAG: ; %bb.0: ; %entry
+; GFX12-TRUE16-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s7, 0x31016000
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s10, s6
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s11, s7
+; GFX12-TRUE16-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX12-TRUE16-SDAG-NEXT: buffer_load_b32 v0, off, s[8:11], null
+; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX12-TRUE16-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.h, v1.l
+; GFX12-TRUE16-SDAG-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX12-TRUE16-SDAG-NEXT: s_endpgm
+;
+; GFX12-TRUE16-GISEL-LABEL: sqrt_v2f16:
+; GFX12-TRUE16-GISEL: ; %bb.0: ; %entry
+; GFX12-TRUE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-GISEL-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX12-TRUE16-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-TRUE16-GISEL-NEXT: v_s_sqrt_f16 s2, s2
+; GFX12-TRUE16-GISEL-NEXT: v_s_sqrt_f16 s3, s3
+; GFX12-TRUE16-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-TRUE16-GISEL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-TRUE16-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-TRUE16-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-TRUE16-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-TRUE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], null
+; GFX12-TRUE16-GISEL-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: sqrt_v2f16:
; GFX12-FAKE16: ; %bb.0: ; %entry
@@ -388,8 +455,3 @@ define amdgpu_ps half @sqrt_f16_v(half %x) {
%result = call half @llvm.sqrt.f16(half %x)
ret half %result
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16-GISEL: {{.*}}
-; GFX11-FAKE16-SDAG: {{.*}}
-; GFX11-TRUE16-GISEL: {{.*}}
-; GFX11-TRUE16-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/lrint.ll b/llvm/test/CodeGen/AMDGPU/lrint.ll
index 6b209b01e58cf..3feaef307f97f 100644
--- a/llvm/test/CodeGen/AMDGPU/lrint.ll
+++ b/llvm/test/CodeGen/AMDGPU/lrint.ll
@@ -87,59 +87,117 @@ entry:
}
define i64 @intrinsic_lrint_i64_f32(float %arg) {
-; GFX9-LABEL: intrinsic_lrint_i64_f32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rndne_f32_e32 v0, v0
-; GFX9-NEXT: s_mov_b32 s4, 0x2f800000
-; GFX9-NEXT: v_mul_f32_e64 v1, |v0|, s4
-; GFX9-NEXT: v_floor_f32_e32 v1, v1
-; GFX9-NEXT: s_mov_b32 s4, 0xcf800000
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v1
-; GFX9-NEXT: v_fma_f32 v1, v1, s4, |v0|
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v1, v3
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v3, vcc
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: intrinsic_lrint_i64_f32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x2f800000
+; GFX9-SDAG-NEXT: v_mul_f32_e64 v1, |v0|, s4
+; GFX9-SDAG-NEXT: v_floor_f32_e32 v1, v1
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0xcf800000
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_fma_f32 v1, v1, s4, |v0|
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v1, v3
+; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
+; GFX9-SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v3, vcc
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: intrinsic_lrint_i64_f32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rndne_f32_e32 v0, v0
-; GFX10-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX10-NEXT: v_floor_f32_e32 v1, v1
-; GFX10-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: intrinsic_lrint_i64_f32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x2f800000
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v2, |v1|, v2
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xcf800000
+; GFX9-GISEL-NEXT: v_fma_f32 v1, v2, v3, |v1|
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v2, v3
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
+; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: intrinsic_lrint_i64_f32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX11-NEXT: v_floor_f32_e32 v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: intrinsic_lrint_i64_f32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-SDAG-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX10-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX10-SDAG-NEXT: v_floor_f32_e32 v1, v1
+; GFX10-SDAG-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX10-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX10-SDAG-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: intrinsic_lrint_i64_f32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX10-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: intrinsic_lrint_i64_f32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX11-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX11-SDAG-NEXT: v_floor_f32_e32 v1, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX11-SDAG-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: intrinsic_lrint_i64_f32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX11-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = tail call i64 @llvm.lrint.i64.f32(float %arg)
ret i64 %res
@@ -232,59 +290,117 @@ entry:
}
define i64 @intrinsic_llrint_i64_f32(float %arg) {
-; GFX9-LABEL: intrinsic_llrint_i64_f32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rndne_f32_e32 v0, v0
-; GFX9-NEXT: s_mov_b32 s4, 0x2f800000
-; GFX9-NEXT: v_mul_f32_e64 v1, |v0|, s4
-; GFX9-NEXT: v_floor_f32_e32 v1, v1
-; GFX9-NEXT: s_mov_b32 s4, 0xcf800000
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v1
-; GFX9-NEXT: v_fma_f32 v1, v1, s4, |v0|
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v1, v3
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v3, vcc
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: intrinsic_llrint_i64_f32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x2f800000
+; GFX9-SDAG-NEXT: v_mul_f32_e64 v1, |v0|, s4
+; GFX9-SDAG-NEXT: v_floor_f32_e32 v1, v1
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0xcf800000
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_fma_f32 v1, v1, s4, |v0|
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v1, v3
+; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
+; GFX9-SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v3, vcc
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: intrinsic_llrint_i64_f32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rndne_f32_e32 v0, v0
-; GFX10-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX10-NEXT: v_floor_f32_e32 v1, v1
-; GFX10-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: intrinsic_llrint_i64_f32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x2f800000
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v2, |v1|, v2
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xcf800000
+; GFX9-GISEL-NEXT: v_fma_f32 v1, v2, v3, |v1|
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v2, v3
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
+; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: intrinsic_llrint_i64_f32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX11-NEXT: v_floor_f32_e32 v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: intrinsic_llrint_i64_f32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-SDAG-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX10-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX10-SDAG-NEXT: v_floor_f32_e32 v1, v1
+; GFX10-SDAG-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX10-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX10-SDAG-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: intrinsic_llrint_i64_f32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX10-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: intrinsic_llrint_i64_f32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX11-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX11-SDAG-NEXT: v_floor_f32_e32 v1, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX11-SDAG-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: intrinsic_llrint_i64_f32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX11-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = tail call i64 @llvm.llrint.i64.f32(float %arg)
ret i64 %res
@@ -520,96 +636,194 @@ entry:
}
define <2 x i64> @intrinsic_lrint_v2i64_v2f32(<2 x float> %arg) {
-; GFX9-LABEL: intrinsic_lrint_v2i64_v2f32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rndne_f32_e32 v0, v0
-; GFX9-NEXT: s_mov_b32 s4, 0x2f800000
-; GFX9-NEXT: v_mul_f32_e64 v2, |v0|, s4
-; GFX9-NEXT: v_floor_f32_e32 v2, v2
-; GFX9-NEXT: s_mov_b32 s5, 0xcf800000
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX9-NEXT: v_fma_f32 v2, v2, s5, |v0|
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v0
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v4
-; GFX9-NEXT: v_xor_b32_e32 v0, v2, v4
-; GFX9-NEXT: v_rndne_f32_e32 v2, v1
-; GFX9-NEXT: v_mul_f32_e64 v1, |v2|, s4
-; GFX9-NEXT: v_floor_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v1
-; GFX9-NEXT: v_fma_f32 v1, v1, s5, |v2|
-; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v4, vcc
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-NEXT: v_xor_b32_e32 v2, v6, v3
-; GFX9-NEXT: v_xor_b32_e32 v4, v5, v3
-; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3
-; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: intrinsic_lrint_v2i64_v2f32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x2f800000
+; GFX9-SDAG-NEXT: v_mul_f32_e64 v2, |v0|, s4
+; GFX9-SDAG-NEXT: v_floor_f32_e32 v2, v2
+; GFX9-SDAG-NEXT: s_mov_b32 s5, 0xcf800000
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v3, v2
+; GFX9-SDAG-NEXT: v_fma_f32 v2, v2, s5, |v0|
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-SDAG-NEXT: v_ashrrev_i32_e32 v4, 31, v0
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v4
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v2, v4
+; GFX9-SDAG-NEXT: v_rndne_f32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mul_f32_e64 v1, |v2|, s4
+; GFX9-SDAG-NEXT: v_floor_f32_e32 v1, v1
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v5, v1
+; GFX9-SDAG-NEXT: v_fma_f32 v1, v1, s5, |v2|
+; GFX9-SDAG-NEXT: v_cvt_u32_f32_e32 v6, v1
+; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4
+; GFX9-SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v6, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v4, v5, v3
+; GFX9-SDAG-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3
+; GFX9-SDAG-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: intrinsic_lrint_v2i64_v2f32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rndne_f32_e32 v0, v0
-; GFX10-NEXT: v_rndne_f32_e32 v1, v1
-; GFX10-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v0|
-; GFX10-NEXT: v_mul_f32_e64 v3, 0x2f800000, |v1|
-; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v0
-; GFX10-NEXT: v_ashrrev_i32_e32 v6, 31, v1
-; GFX10-NEXT: v_floor_f32_e32 v2, v2
-; GFX10-NEXT: v_floor_f32_e32 v3, v3
-; GFX10-NEXT: v_fma_f32 v4, 0xcf800000, v2, |v0|
-; GFX10-NEXT: v_fma_f32 v0, 0xcf800000, v3, |v1|
-; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v4
-; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v5
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v6
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_xor_b32_e32 v4, v0, v6
-; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v5
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v2, v5, vcc_lo
-; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, v4, v6
-; GFX10-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: intrinsic_lrint_v2i64_v2f32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x2f800000
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v4, |v2|, v3
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v4, v4
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xcf800000
+; GFX9-GISEL-NEXT: v_fma_f32 v2, v4, v5, |v2|
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v0
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v2, v6
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v4, v6
+; GFX9-GISEL-NEXT: v_rndne_f32_e32 v4, v1
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v1, v4
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v3, |v1|, v3
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v3, v3
+; GFX9-GISEL-NEXT: v_fma_f32 v1, v3, v5, |v1|
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v5, v1
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v6
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v4
+; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v6, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v5, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v4
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v4
+; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: intrinsic_lrint_v2i64_v2f32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11-NEXT: v_rndne_f32_e32 v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v0|
-; GFX11-NEXT: v_mul_f32_e64 v3, 0x2f800000, |v1|
-; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v0
-; GFX11-NEXT: v_ashrrev_i32_e32 v6, 31, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_floor_f32_e32 v2, v2
-; GFX11-NEXT: v_floor_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fma_f32 v4, 0xcf800000, v2, |v0|
-; GFX11-NEXT: v_fma_f32 v0, 0xcf800000, v3, |v1|
-; GFX11-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v4
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v5
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_xor_b32_e32 v4, v0, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v5
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v1, null, v2, v5, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_co_u32 v2, vcc_lo, v4, v6
-; GFX11-NEXT: v_sub_co_ci_u32_e64 v3, null, v3, v6, vcc_lo
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: intrinsic_lrint_v2i64_v2f32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-SDAG-NEXT: v_rndne_f32_e32 v1, v1
+; GFX10-SDAG-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v0|
+; GFX10-SDAG-NEXT: v_mul_f32_e64 v3, 0x2f800000, |v1|
+; GFX10-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v0
+; GFX10-SDAG-NEXT: v_ashrrev_i32_e32 v6, 31, v1
+; GFX10-SDAG-NEXT: v_floor_f32_e32 v2, v2
+; GFX10-SDAG-NEXT: v_floor_f32_e32 v3, v3
+; GFX10-SDAG-NEXT: v_fma_f32 v4, 0xcf800000, v2, |v0|
+; GFX10-SDAG-NEXT: v_fma_f32 v0, 0xcf800000, v3, |v1|
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v4
+; GFX10-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v5
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v6
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v4, v0, v6
+; GFX10-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v5
+; GFX10-SDAG-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v2, v5, vcc_lo
+; GFX10-SDAG-NEXT: v_sub_co_u32 v2, vcc_lo, v4, v6
+; GFX10-SDAG-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: intrinsic_lrint_v2i64_v2f32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_rndne_f32_e32 v1, v1
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v2, v0
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v3, v1
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v0
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v4, 0x2f800000, |v2|
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v5, 0x2f800000, |v3|
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v4, v4
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v5, v5
+; GFX10-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v4, |v2|
+; GFX10-GISEL-NEXT: v_fma_f32 v0, 0xcf800000, v5, |v3|
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v1
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v4
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v4, v5
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v6
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v5, v0, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v4, v4, v3
+; GFX10-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v6
+; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v2, v6, vcc_lo
+; GFX10-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v5, v3
+; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, v4, v3, vcc_lo
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: intrinsic_lrint_v2i64_v2f32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-SDAG-NEXT: v_rndne_f32_e32 v1, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v0|
+; GFX11-SDAG-NEXT: v_mul_f32_e64 v3, 0x2f800000, |v1|
+; GFX11-SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v0
+; GFX11-SDAG-NEXT: v_ashrrev_i32_e32 v6, 31, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_floor_f32_e32 v2, v2
+; GFX11-SDAG-NEXT: v_floor_f32_e32 v3, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_fma_f32 v4, 0xcf800000, v2, |v0|
+; GFX11-SDAG-NEXT: v_fma_f32 v0, 0xcf800000, v3, |v1|
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v4
+; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v5
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v6
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v4, v0, v6
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v5
+; GFX11-SDAG-NEXT: v_sub_co_ci_u32_e64 v1, null, v2, v5, vcc_lo
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_co_u32 v2, vcc_lo, v4, v6
+; GFX11-SDAG-NEXT: v_sub_co_ci_u32_e64 v3, null, v3, v6, vcc_lo
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: intrinsic_lrint_v2i64_v2f32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-GISEL-NEXT: v_rndne_f32_e32 v1, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v2, v0
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v3, v1
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v4, 0x2f800000, |v2|
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v5, 0x2f800000, |v3|
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v4, v4
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v5, v5
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v4, |v2|
+; GFX11-GISEL-NEXT: v_fma_f32 v0, 0xcf800000, v5, |v3|
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v4
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v4, v5
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v6
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v5, v0, v3
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v4, v4, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v6
+; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v2, v6, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v5, v3
+; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v3, null, v4, v3, vcc_lo
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = tail call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> %arg)
ret <2 x i64> %res
diff --git a/llvm/test/CodeGen/AMDGPU/lround.ll b/llvm/test/CodeGen/AMDGPU/lround.ll
index 4f9cf3078f4c7..a598dc91c4040 100644
--- a/llvm/test/CodeGen/AMDGPU/lround.ll
+++ b/llvm/test/CodeGen/AMDGPU/lround.ll
@@ -116,13 +116,14 @@ define i32 @intrinsic_lround_i32_f64(double %arg) {
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX9-GISEL-NEXT: s_brev_b32 s4, -2
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v6, -2
; GFX9-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x3ff00000
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX9-GISEL-NEXT: v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
-; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-GISEL-NEXT: v_bfi_b32 v1, s4, v0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v4, v6, v1
; GFX9-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
; GFX9-GISEL-NEXT: v_cvt_i32_f64_e32 v0, v[0:1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -145,10 +146,11 @@ define i32 @intrinsic_lround_i32_f64(double %arg) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; GFX10-GISEL-NEXT: v_cmp_ge_f64_e64 s4, |v[4:5]|, 0.5
-; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 0x3ff00000, s4
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: v_cmp_ge_f64_e64 s4, |v[4:5]|, 0.5
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 0x3ff00000, s4
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v4, v1
; GFX10-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_cvt_i32_f64_e32 v0, v[0:1]
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -173,13 +175,13 @@ define i32 @intrinsic_lround_i32_f64(double %arg) {
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_and_b32 v1, 0x80000000, v1
; GFX11-GISEL-NEXT: v_cmp_ge_f64_e64 s0, |v[4:5]|, 0.5
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 0x3ff00000, s0
-; GFX11-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v0, v1
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 0x3ff00000, s0
+; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v4, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
; GFX11-GISEL-NEXT: v_cvt_i32_f64_e32 v0, v[0:1]
@@ -222,20 +224,21 @@ define i64 @intrinsic_lround_i64_f32(float %arg) {
; GFX9-GISEL-NEXT: v_sub_f32_e32 v2, v0, v1
; GFX9-GISEL-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, 0.5
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[4:5]
-; GFX9-GISEL-NEXT: s_brev_b32 s4, -2
-; GFX9-GISEL-NEXT: v_bfi_b32 v0, s4, v2, v0
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v3, -2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v2, v3, v0
; GFX9-GISEL-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX9-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x2f800000
-; GFX9-GISEL-NEXT: v_mul_f32_e64 v1, |v0|, s4
-; GFX9-GISEL-NEXT: v_floor_f32_e32 v1, v1
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0xcf800000
-; GFX9-GISEL-NEXT: v_fma_f32 v2, v1, s4, |v0|
-; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x2f800000
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v2, |v1|, v2
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xcf800000
+; GFX9-GISEL-NEXT: v_fma_f32 v1, v2, v3, |v1|
; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v2, v3
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v2, v3
; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -267,19 +270,20 @@ define i64 @intrinsic_lround_i64_f32(float %arg) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_trunc_f32_e32 v1, v0
; GFX10-GISEL-NEXT: v_sub_f32_e32 v2, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
; GFX10-GISEL-NEXT: v_cmp_ge_f32_e64 s4, |v2|, 0.5
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s4
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0x7fffffff, v2, v0
; GFX10-GISEL-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX10-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX10-GISEL-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v1, v0
; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX10-GISEL-NEXT: v_floor_f32_e32 v1, v1
-; GFX10-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX10-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX10-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -317,27 +321,28 @@ define i64 @intrinsic_lround_i64_f32(float %arg) {
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_trunc_f32_e32 v1, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_sub_f32_e32 v2, v0, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
; GFX11-GISEL-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0
-; GFX11-GISEL-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0x7fffffff, v2, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v1, v0
; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX11-GISEL-NEXT: v_floor_f32_e32 v1, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -375,20 +380,22 @@ define i64 @intrinsic_lround_i64_f64(double %arg) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x3ff00000
-; GFX9-GISEL-NEXT: s_brev_b32 s4, -2
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xc1f00000
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v7, -2
; GFX9-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xc1f00000
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
-; GFX9-GISEL-NEXT: v_bfi_b32 v1, s4, v4, v1
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v4, v7, v1
; GFX9-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0xffe0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x3df00000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
-; GFX9-GISEL-NEXT: v_ldexp_f64 v[2:3], v[0:1], s4
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
; GFX9-GISEL-NEXT: v_floor_f64_e32 v[2:3], v[2:3]
-; GFX9-GISEL-NEXT: v_fma_f64 v[0:1], v[2:3], s[4:5], v[0:1]
+; GFX9-GISEL-NEXT: v_fma_f64 v[0:1], v[2:3], v[4:5], v[0:1]
; GFX9-GISEL-NEXT: v_cvt_u32_f64_e32 v0, v[0:1]
; GFX9-GISEL-NEXT: v_cvt_i32_f64_e32 v1, v[2:3]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -416,13 +423,14 @@ define i64 @intrinsic_lround_i64_f64(double %arg) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: v_cmp_ge_f64_e64 s4, |v[4:5]|, 0.5
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 0x3ff00000, s4
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v4, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v4, v1
; GFX10-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
-; GFX10-GISEL-NEXT: v_ldexp_f64 v[2:3], v[0:1], 0xffffffe0
+; GFX10-GISEL-NEXT: v_mul_f64 v[2:3], 0x3df00000, v[0:1]
; GFX10-GISEL-NEXT: v_floor_f64_e32 v[2:3], v[2:3]
; GFX10-GISEL-NEXT: v_fma_f64 v[0:1], 0xc1f00000, v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_cvt_u32_f64_e32 v0, v[0:1]
@@ -458,16 +466,16 @@ define i64 @intrinsic_lround_i64_f64(double %arg) {
; GFX11-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_and_b32 v1, 0x80000000, v1
; GFX11-GISEL-NEXT: v_cmp_ge_f64_e64 s0, |v[4:5]|, 0.5
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 0x3ff00000, s0
-; GFX11-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v4, v1
+; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v4, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
; GFX11-GISEL-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_ldexp_f64 v[2:3], v[0:1], 0xffffffe0
+; GFX11-GISEL-NEXT: v_mul_f64 v[2:3], 0x3df00000, v[0:1]
; GFX11-GISEL-NEXT: v_floor_f64_e32 v[2:3], v[2:3]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_fma_f64 v[0:1], 0xc1f00000, v[2:3], v[0:1]
@@ -512,20 +520,21 @@ define i64 @intrinsic_llround_i64_f32(float %arg) {
; GFX9-GISEL-NEXT: v_sub_f32_e32 v2, v0, v1
; GFX9-GISEL-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, 0.5
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[4:5]
-; GFX9-GISEL-NEXT: s_brev_b32 s4, -2
-; GFX9-GISEL-NEXT: v_bfi_b32 v0, s4, v2, v0
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v3, -2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v2, v3, v0
; GFX9-GISEL-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX9-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x2f800000
-; GFX9-GISEL-NEXT: v_mul_f32_e64 v1, |v0|, s4
-; GFX9-GISEL-NEXT: v_floor_f32_e32 v1, v1
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0xcf800000
-; GFX9-GISEL-NEXT: v_fma_f32 v2, v1, s4, |v0|
-; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x2f800000
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v2, |v1|, v2
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xcf800000
+; GFX9-GISEL-NEXT: v_fma_f32 v1, v2, v3, |v1|
; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v2, v3
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v2, v3
; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -557,19 +566,20 @@ define i64 @intrinsic_llround_i64_f32(float %arg) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_trunc_f32_e32 v1, v0
; GFX10-GISEL-NEXT: v_sub_f32_e32 v2, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
; GFX10-GISEL-NEXT: v_cmp_ge_f32_e64 s4, |v2|, 0.5
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s4
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0x7fffffff, v2, v0
; GFX10-GISEL-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX10-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX10-GISEL-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v1, v0
; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX10-GISEL-NEXT: v_floor_f32_e32 v1, v1
-; GFX10-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX10-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX10-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -607,27 +617,28 @@ define i64 @intrinsic_llround_i64_f32(float %arg) {
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_trunc_f32_e32 v1, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_sub_f32_e32 v2, v0, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
; GFX11-GISEL-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0
-; GFX11-GISEL-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0x7fffffff, v2, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v1, v0
; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX11-GISEL-NEXT: v_floor_f32_e32 v1, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v1|
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v2, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_fma_f32 v1, 0xcf800000, v2, |v1|
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -665,20 +676,22 @@ define i64 @intrinsic_llround_i64_f64(double %arg) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x3ff00000
-; GFX9-GISEL-NEXT: s_brev_b32 s4, -2
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xc1f00000
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v7, -2
; GFX9-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xc1f00000
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v6, vcc
-; GFX9-GISEL-NEXT: v_bfi_b32 v1, s4, v4, v1
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v4, v7, v1
; GFX9-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0xffe0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x3df00000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-GISEL-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
-; GFX9-GISEL-NEXT: v_ldexp_f64 v[2:3], v[0:1], s4
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
; GFX9-GISEL-NEXT: v_floor_f64_e32 v[2:3], v[2:3]
-; GFX9-GISEL-NEXT: v_fma_f64 v[0:1], v[2:3], s[4:5], v[0:1]
+; GFX9-GISEL-NEXT: v_fma_f64 v[0:1], v[2:3], v[4:5], v[0:1]
; GFX9-GISEL-NEXT: v_cvt_u32_f64_e32 v0, v[0:1]
; GFX9-GISEL-NEXT: v_cvt_i32_f64_e32 v1, v[2:3]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -706,13 +719,14 @@ define i64 @intrinsic_llround_i64_f64(double %arg) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: v_cmp_ge_f64_e64 s4, |v[4:5]|, 0.5
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 0x3ff00000, s4
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v4, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v4, v1
; GFX10-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
-; GFX10-GISEL-NEXT: v_ldexp_f64 v[2:3], v[0:1], 0xffffffe0
+; GFX10-GISEL-NEXT: v_mul_f64 v[2:3], 0x3df00000, v[0:1]
; GFX10-GISEL-NEXT: v_floor_f64_e32 v[2:3], v[2:3]
; GFX10-GISEL-NEXT: v_fma_f64 v[0:1], 0xc1f00000, v[2:3], v[0:1]
; GFX10-GISEL-NEXT: v_cvt_u32_f64_e32 v0, v[0:1]
@@ -748,16 +762,16 @@ define i64 @intrinsic_llround_i64_f64(double %arg) {
; GFX11-GISEL-NEXT: v_trunc_f64_e32 v[2:3], v[0:1]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_add_f64 v[4:5], v[0:1], -v[2:3]
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_and_b32 v1, 0x80000000, v1
; GFX11-GISEL-NEXT: v_cmp_ge_f64_e64 s0, |v[4:5]|, 0.5
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 0x3ff00000, s0
-; GFX11-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v4, v1
+; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v4, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1]
; GFX11-GISEL-NEXT: v_trunc_f64_e32 v[0:1], v[0:1]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_ldexp_f64 v[2:3], v[0:1], 0xffffffe0
+; GFX11-GISEL-NEXT: v_mul_f64 v[2:3], 0x3df00000, v[0:1]
; GFX11-GISEL-NEXT: v_floor_f64_e32 v[2:3], v[2:3]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_fma_f64 v[0:1], 0xc1f00000, v[2:3], v[0:1]
@@ -1196,39 +1210,41 @@ define <2 x i64> @intrinsic_lround_v2i64_v2f32(<2 x float> %arg) {
; GFX9-GISEL-NEXT: v_sub_f32_e32 v3, v0, v2
; GFX9-GISEL-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, 0.5
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, s[4:5]
-; GFX9-GISEL-NEXT: s_brev_b32 s6, -2
-; GFX9-GISEL-NEXT: v_bfi_b32 v0, s6, v3, v0
+; GFX9-GISEL-NEXT: v_bfrev_b32_e32 v4, -2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v3, v4, v0
; GFX9-GISEL-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX9-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0x2f800000
-; GFX9-GISEL-NEXT: v_mul_f32_e64 v2, |v0|, s7
-; GFX9-GISEL-NEXT: v_floor_f32_e32 v2, v2
-; GFX9-GISEL-NEXT: s_mov_b32 s8, 0xcf800000
-; GFX9-GISEL-NEXT: v_fma_f32 v3, v2, s8, |v0|
-; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v0
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x2f800000
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v5, |v2|, v3
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v5, v5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0xcf800000
+; GFX9-GISEL-NEXT: v_fma_f32 v2, v5, v6, |v2|
; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v3, v4
-; GFX9-GISEL-NEXT: v_trunc_f32_e32 v3, v1
-; GFX9-GISEL-NEXT: v_sub_f32_e32 v5, v1, v3
-; GFX9-GISEL-NEXT: v_cmp_ge_f32_e64 s[4:5], |v5|, 0.5
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1.0, s[4:5]
-; GFX9-GISEL-NEXT: v_bfi_b32 v1, s6, v5, v1
-; GFX9-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX9-GISEL-NEXT: v_trunc_f32_e32 v3, v1
-; GFX9-GISEL-NEXT: v_mul_f32_e64 v1, |v3|, s7
-; GFX9-GISEL-NEXT: v_floor_f32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_fma_f32 v5, v1, s8, |v3|
; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v5, v5
-; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v6, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v4
-; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4
-; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v3
-; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v4, vcc
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v5, v3
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v4, v6, v3
-; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3
-; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v0
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v2, v7
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v5, v7
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v5, v1
+; GFX9-GISEL-NEXT: v_sub_f32_e32 v8, v1, v5
+; GFX9-GISEL-NEXT: v_cmp_ge_f32_e64 s[4:5], |v8|, 0.5
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1.0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v8, v4, v1
+; GFX9-GISEL-NEXT: v_add_f32_e32 v4, v5, v1
+; GFX9-GISEL-NEXT: v_trunc_f32_e32 v1, v4
+; GFX9-GISEL-NEXT: v_mul_f32_e64 v3, |v1|, v3
+; GFX9-GISEL-NEXT: v_floor_f32_e32 v3, v3
+; GFX9-GISEL-NEXT: v_fma_f32 v1, v3, v6, |v1|
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v5, v1
+; GFX9-GISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v7
+; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v4
+; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v7, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v5, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v4
+; GFX9-GISEL-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v4
+; GFX9-GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: intrinsic_lround_v2i64_v2f32:
@@ -1277,36 +1293,38 @@ define <2 x i64> @intrinsic_lround_v2i64_v2f32(<2 x float> %arg) {
; GFX10-GISEL-NEXT: v_trunc_f32_e32 v3, v1
; GFX10-GISEL-NEXT: v_sub_f32_e32 v4, v0, v2
; GFX10-GISEL-NEXT: v_sub_f32_e32 v5, v1, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX10-GISEL-NEXT: v_cmp_ge_f32_e64 s4, |v4|, 0.5
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1.0, s4
; GFX10-GISEL-NEXT: v_cmp_ge_f32_e64 s4, |v5|, 0.5
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0x7fffffff, v4, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1.0, s4
; GFX10-GISEL-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX10-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1
-; GFX10-GISEL-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v5, v1
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v2, v0
; GFX10-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX10-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v0|
-; GFX10-GISEL-NEXT: v_trunc_f32_e32 v1, v1
-; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v0
-; GFX10-GISEL-NEXT: v_floor_f32_e32 v2, v2
-; GFX10-GISEL-NEXT: v_mul_f32_e64 v3, 0x2f800000, |v1|
-; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v1
-; GFX10-GISEL-NEXT: v_fma_f32 v4, 0xcf800000, v2, |v0|
-; GFX10-GISEL-NEXT: v_floor_f32_e32 v3, v3
-; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX10-GISEL-NEXT: v_fma_f32 v0, 0xcf800000, v3, |v1|
-; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v4
-; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, v2, v5
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v0
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v4, 0x2f800000, |v2|
+; GFX10-GISEL-NEXT: v_trunc_f32_e32 v3, v1
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v4, v4
+; GFX10-GISEL-NEXT: v_mul_f32_e64 v5, 0x2f800000, |v3|
+; GFX10-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v4, |v2|
+; GFX10-GISEL-NEXT: v_floor_f32_e32 v5, v5
+; GFX10-GISEL-NEXT: v_fma_f32 v0, 0xcf800000, v5, |v3|
+; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v1
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v4
+; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v4, v5
; GFX10-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v3, v3, v6
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v4, v0, v6
-; GFX10-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v5
-; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v2, v5, vcc_lo
-; GFX10-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v4, v6
-; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v6
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v4, v4, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v5, v0, v3
+; GFX10-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v6
+; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v2, v6, vcc_lo
+; GFX10-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v5, v3
+; GFX10-GISEL-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, v4, v3, vcc_lo
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: intrinsic_lround_v2i64_v2f32:
@@ -1364,49 +1382,51 @@ define <2 x i64> @intrinsic_lround_v2i64_v2f32(<2 x float> %arg) {
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_trunc_f32_e32 v2, v0
; GFX11-GISEL-NEXT: v_trunc_f32_e32 v3, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_dual_sub_f32 v4, v0, v2 :: v_dual_sub_f32 v5, v1, v3
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX11-GISEL-NEXT: v_cmp_ge_f32_e64 s0, |v4|, 0.5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1.0, s0
; GFX11-GISEL-NEXT: v_cmp_ge_f32_e64 s0, |v5|, 0.5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1.0, s0
+; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0x7fffffff, v5, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1
-; GFX11-GISEL-NEXT: v_dual_add_f32 v0, v2, v0 :: v_dual_add_f32 v1, v3, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_trunc_f32_e32 v0, v0
-; GFX11-GISEL-NEXT: v_trunc_f32_e32 v1, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_mul_f32_e64 v2, 0x2f800000, |v0|
-; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v0
-; GFX11-GISEL-NEXT: v_mul_f32_e64 v3, 0x2f800000, |v1|
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_floor_f32_e32 v2, v2
-; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v1
-; GFX11-GISEL-NEXT: v_floor_f32_e32 v3, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_fma_f32 v4, 0xcf800000, v2, |v0|
-; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX11-GISEL-NEXT: v_fma_f32 v0, 0xcf800000, v3, |v1|
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v4
-; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_dual_add_f32 v1, v3, v1 :: v_dual_and_b32 v0, 0x80000000, v0
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0x7fffffff, v4, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v3, v1
+; GFX11-GISEL-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v5, 0x2f800000, |v3|
+; GFX11-GISEL-NEXT: v_trunc_f32_e32 v2, v0
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v5, v5
+; GFX11-GISEL-NEXT: v_mul_f32_e64 v4, 0x2f800000, |v2|
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_fma_f32 v0, 0xcf800000, v5, |v3|
+; GFX11-GISEL-NEXT: v_floor_f32_e32 v4, v4
+; GFX11-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v3, v3, v6
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v4, v0, v6
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v5
-; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v2, v5, vcc_lo
+; GFX11-GISEL-NEXT: v_fma_f32 v2, 0xcf800000, v4, |v2|
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v2, v4
+; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v4, v5
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v5, v0, v3
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v4, v4, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v4, v6
-; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v3, null, v3, v6, vcc_lo
+; GFX11-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v1, v6
+; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v2, v6, vcc_lo
+; GFX11-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v5, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_co_ci_u32_e64 v3, null, v4, v3, vcc_lo
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = tail call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> %arg)
diff --git a/llvm/test/CodeGen/AMDGPU/roundeven.ll b/llvm/test/CodeGen/AMDGPU/roundeven.ll
index 697f9b7524cb0..7230f2e8782dd 100644
--- a/llvm/test/CodeGen/AMDGPU/roundeven.ll
+++ b/llvm/test/CodeGen/AMDGPU/roundeven.ll
@@ -1107,15 +1107,14 @@ define double @v_roundeven_f64(double %x) {
; GFX6-LABEL: v_roundeven_f64:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_brev_b32 s6, -2
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x43300000
-; GFX6-NEXT: v_bfi_b32 v3, s6, v2, v1
+; GFX6-NEXT: v_and_b32_e32 v3, 0x80000000, v1
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: v_or_b32_e32 v3, 0x43300000, v3
; GFX6-NEXT: v_add_f64 v[4:5], v[0:1], v[2:3]
-; GFX6-NEXT: s_mov_b32 s4, -1
-; GFX6-NEXT: s_mov_b32 s5, 0x432fffff
+; GFX6-NEXT: v_mov_b32_e32 v6, -1
+; GFX6-NEXT: v_mov_b32_e32 v7, 0x432fffff
; GFX6-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -1191,18 +1190,17 @@ define double @v_roundeven_f64_fneg(double %x) {
; GFX6-LABEL: v_roundeven_f64_fneg:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_xor_b32_e32 v6, 0x80000000, v1
-; GFX6-NEXT: s_brev_b32 s4, -2
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x43300000
-; GFX6-NEXT: v_bfi_b32 v3, s4, v2, v6
+; GFX6-NEXT: v_xor_b32_e32 v8, 0x80000000, v1
+; GFX6-NEXT: v_and_b32_e32 v3, 0x80000000, v8
; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: v_or_b32_e32 v3, 0x43300000, v3
; GFX6-NEXT: v_add_f64 v[4:5], -v[0:1], v[2:3]
-; GFX6-NEXT: s_mov_b32 s4, -1
-; GFX6-NEXT: s_mov_b32 s5, 0x432fffff
+; GFX6-NEXT: v_mov_b32_e32 v6, -1
+; GFX6-NEXT: v_mov_b32_e32 v7, 0x432fffff
; GFX6-NEXT: v_add_f64 v[2:3], v[4:5], -v[2:3]
-; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[6:7]
; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v8, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_roundeven_f64_fneg:
@@ -1278,21 +1276,21 @@ define <2 x double> @v_roundeven_v2f64(<2 x double> %x) {
; GFX6-LABEL: v_roundeven_v2f64:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_brev_b32 s6, -2
-; GFX6-NEXT: v_mov_b32_e32 v8, 0x43300000
-; GFX6-NEXT: v_bfi_b32 v5, s6, v8, v1
+; GFX6-NEXT: v_and_b32_e32 v5, 0x80000000, v1
; GFX6-NEXT: v_mov_b32_e32 v4, 0
+; GFX6-NEXT: v_or_b32_e32 v5, 0x43300000, v5
; GFX6-NEXT: v_add_f64 v[6:7], v[0:1], v[4:5]
-; GFX6-NEXT: s_mov_b32 s4, -1
-; GFX6-NEXT: s_mov_b32 s5, 0x432fffff
+; GFX6-NEXT: v_mov_b32_e32 v8, -1
+; GFX6-NEXT: v_mov_b32_e32 v9, 0x432fffff
; GFX6-NEXT: v_add_f64 v[5:6], v[6:7], -v[4:5]
-; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[0:1]|, v[8:9]
; GFX6-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX6-NEXT: v_bfi_b32 v5, s6, v8, v3
-; GFX6-NEXT: v_add_f64 v[7:8], v[2:3], v[4:5]
+; GFX6-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; GFX6-NEXT: v_or_b32_e32 v5, 0x43300000, v5
+; GFX6-NEXT: v_add_f64 v[10:11], v[2:3], v[4:5]
; GFX6-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX6-NEXT: v_add_f64 v[4:5], v[7:8], -v[4:5]
-; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; GFX6-NEXT: v_add_f64 v[4:5], v[10:11], -v[4:5]
+; GFX6-NEXT: v_cmp_gt_f64_e64 vcc, |v[2:3]|, v[8:9]
; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
index 2835d67704005..c4446f9a7edae 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
@@ -351,47 +351,91 @@ entry:
}
define amdgpu_ps <2 x float> @flat_atomicrmw_b64_rtn_idxprom(ptr align 8 inreg %p, i32 %idx) {
-; GCN-LABEL: flat_atomicrmw_b64_rtn_idxprom:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 3, s[0:1]
-; GCN-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GCN-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GCN-NEXT: s_xor_b32 s0, exec_lo, s0
-; GCN-NEXT: s_cbranch_execnz .LBB21_3
-; GCN-NEXT: ; %bb.1: ; %Flow
-; GCN-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GCN-NEXT: s_cbranch_execnz .LBB21_4
-; GCN-NEXT: .LBB21_2: ; %atomicrmw.phi
-; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
-; GCN-NEXT: s_branch .LBB21_5
-; GCN-NEXT: .LBB21_3: ; %atomicrmw.global
-; GCN-NEXT: v_mov_b64_e32 v[0:1], 1
-; GCN-NEXT: flat_atomic_add_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_wait_xcnt 0x0
-; GCN-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GCN-NEXT: s_cbranch_execz .LBB21_2
-; GCN-NEXT: .LBB21_4: ; %atomicrmw.private
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
-; GCN-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GCN-NEXT: scratch_load_b64 v[0:1], v4, off
-; GCN-NEXT: s_wait_loadcnt 0x0
-; GCN-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]
-; GCN-NEXT: scratch_store_b64 v4, v[2:3], off
-; GCN-NEXT: s_wait_xcnt 0x0
-; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GCN-NEXT: s_branch .LBB21_5
-; GCN-NEXT: .LBB21_5:
+; SDAG-LABEL: flat_atomicrmw_b64_rtn_idxprom:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 3, s[0:1]
+; SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; SDAG-NEXT: s_xor_b32 s0, exec_lo, s0
+; SDAG-NEXT: s_cbranch_execnz .LBB21_3
+; SDAG-NEXT: ; %bb.1: ; %Flow
+; SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; SDAG-NEXT: s_cbranch_execnz .LBB21_4
+; SDAG-NEXT: .LBB21_2: ; %atomicrmw.phi
+; SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: s_branch .LBB21_5
+; SDAG-NEXT: .LBB21_3: ; %atomicrmw.global
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], 1
+; SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_wait_xcnt 0x0
+; SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
+; SDAG-NEXT: s_cbranch_execz .LBB21_2
+; SDAG-NEXT: .LBB21_4: ; %atomicrmw.private
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]
+; SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
+; SDAG-NEXT: s_wait_xcnt 0x0
+; SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; SDAG-NEXT: s_branch .LBB21_5
+; SDAG-NEXT: .LBB21_5:
+;
+; GISEL-LABEL: flat_atomicrmw_b64_rtn_idxprom:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GISEL-NEXT: v_lshl_add_u64 v[4:5], v[2:3], 3, s[0:1]
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GISEL-NEXT: s_cbranch_execnz .LBB21_3
+; GISEL-NEXT: ; %bb.1: ; %Flow
+; GISEL-NEXT: s_and_not1_saveexec_b32 s0, s2
+; GISEL-NEXT: s_cbranch_execnz .LBB21_4
+; GISEL-NEXT: .LBB21_2: ; %atomicrmw.phi
+; GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: s_branch .LBB21_5
+; GISEL-NEXT: .LBB21_3: ; %atomicrmw.global
+; GISEL-NEXT: v_mov_b64_e32 v[0:1], 1
+; GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GISEL-NEXT: flat_atomic_add_u64 v[0:1], v2, v[0:1], s[0:1] scale_offset th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GISEL-NEXT: s_wait_xcnt 0x0
+; GISEL-NEXT: s_and_not1_saveexec_b32 s0, s2
+; GISEL-NEXT: s_cbranch_execz .LBB21_2
+; GISEL-NEXT: .LBB21_4: ; %atomicrmw.private
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GISEL-NEXT: scratch_load_b64 v[0:1], v4, off
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]
+; GISEL-NEXT: scratch_store_b64 v4, v[2:3], off
+; GISEL-NEXT: s_wait_xcnt 0x0
+; GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GISEL-NEXT: s_branch .LBB21_5
+; GISEL-NEXT: .LBB21_5:
entry:
%idxprom = sext i32 %idx to i64
%arrayidx = getelementptr inbounds i64, ptr %p, i64 %idxprom
@@ -402,9 +446,7 @@ entry:
!0 = !{i32 0, i32 1024}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GISEL: {{.*}}
; GISEL-FAKE16: {{.*}}
; GISEL-REAL16: {{.*}}
-; SDAG: {{.*}}
; SDAG-FAKE16: {{.*}}
; SDAG-REAL16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
index fa70cf04c9f37..4d6afd13fb207 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
@@ -59,17 +59,30 @@ define amdgpu_ps float @global_load_b32_idxprom_wrong_stride(ptr addrspace(1) al
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
;
-; GFX13-LABEL: global_load_b32_idxprom_wrong_stride:
-; GFX13: ; %bb.0: ; %entry
-; GFX13-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_lshlrev_b64_e32 v[0:1], 3, v[0:1]
-; GFX13-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX13-NEXT: global_load_b32 v0, v[0:1], off
-; GFX13-NEXT: s_wait_loadcnt 0x0
-; GFX13-NEXT: ; return to shader part epilog
+; GFX13-SDAG-LABEL: global_load_b32_idxprom_wrong_stride:
+; GFX13-SDAG: ; %bb.0: ; %entry
+; GFX13-SDAG-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], 3, v[0:1]
+; GFX13-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX13-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX13-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX13-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX13-GISEL-LABEL: global_load_b32_idxprom_wrong_stride:
+; GFX13-GISEL: ; %bb.0: ; %entry
+; GFX13-GISEL-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_mov_b32 v3, s1
+; GFX13-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 3, v[0:1]
+; GFX13-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX13-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX13-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX13-GISEL-NEXT: ; return to shader part epilog
entry:
%idxprom = sext i32 %idx to i64
%arrayidx = getelementptr inbounds <2 x float>, ptr addrspace(1) %p, i64 %idxprom
@@ -535,6 +548,3 @@ entry:
}
!0 = !{i32 0, i32 1024}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX13-GISEL: {{.*}}
-; GFX13-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
index f1639caed1307..38a4d9113723a 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
@@ -20,17 +20,30 @@ entry:
; 'i32 %idx' is a signed index while SMRD soffset is unsigned, thus it is not selected.
define amdgpu_ps float @s_load_b32_idx32(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
-; GCN-LABEL: s_load_b32_idx32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: s_ashr_i32 s3, s2, 31
-; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
-; GCN-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT: s_load_b32 s0, s[0:1], 0x0 nv
-; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: ; return to shader part epilog
+; SDAG-LABEL: s_load_b32_idx32:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_ashr_i32 s3, s2, 31
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; SDAG-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
+; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; SDAG-NEXT: s_load_b32 s0, s[0:1], 0x0 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-LABEL: s_load_b32_idx32:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_ashr_i32 s3, s2, 31
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
+; GISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GISEL-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GISEL-NEXT: s_load_b32 s0, s[0:1], 0x0 nv
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: ; return to shader part epilog
entry:
%arrayidx = getelementptr inbounds float, ptr addrspace(4) %p, i32 %idx
%ret = load float, ptr addrspace(4) %arrayidx, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
index 252a7d87561d8..71df27c92538d 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
@@ -503,10 +503,21 @@ define amdgpu_ps <2 x half> @s_constained_fadd_v2f16_fpexcept_strict(<2 x half>
; GFX11-NEXT: v_pk_add_f16 v0, s2, s3
; GFX11-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: s_constained_fadd_v2f16_fpexcept_strict:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_pk_add_f16 v0, s2, s3
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: s_constained_fadd_v2f16_fpexcept_strict:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_pk_add_f16 v0, s2, s3
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: s_constained_fadd_v2f16_fpexcept_strict:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s2, 16
+; GFX12-GISEL-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GISEL-NEXT: s_add_f16 s2, s2, s3
+; GFX12-GISEL-NEXT: s_add_f16 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call <2 x half> @llvm.experimental.constrained.fadd.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret <2 x half> %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
index 280ee9fe4d8f0..2925ebd74bbcd 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
@@ -578,10 +578,21 @@ define amdgpu_ps <2 x half> @s_constained_fmul_v2f16_fpexcept_strict(<2 x half>
; GFX10PLUS-NEXT: v_pk_mul_f16 v0, s2, s3
; GFX10PLUS-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: s_constained_fmul_v2f16_fpexcept_strict:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_pk_mul_f16 v0, s2, s3
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: s_constained_fmul_v2f16_fpexcept_strict:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_pk_mul_f16 v0, s2, s3
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: s_constained_fmul_v2f16_fpexcept_strict:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s2, 16
+; GFX12-GISEL-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-GISEL-NEXT: s_mul_f16 s2, s2, s3
+; GFX12-GISEL-NEXT: s_mul_f16 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call <2 x half> @llvm.experimental.constrained.fmul.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret <2 x half> %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 01c4aa583981b..4daa09214eaa1 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -841,17 +841,30 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
; GFX11-GISEL-NEXT: v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_lshr_b32 s0, s3, 16
-; GFX12-NEXT: s_lshr_b32 s1, s2, 16
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_3)
-; GFX12-NEXT: s_sub_f16 s0, s1, s0
-; GFX12-NEXT: s_sub_f16 s1, s2, s3
-; GFX12-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX12-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: s_sub_f16 s0, s1, s0
+; GFX12-SDAG-NEXT: s_sub_f16 s1, s2, s3
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s3, 16
+; GFX12-GISEL-NEXT: s_xor_b32 s1, s3, 0x8000
+; GFX12-GISEL-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-GISEL-NEXT: s_add_f16 s1, s2, s1
+; GFX12-GISEL-NEXT: s_add_f16 s0, s3, s0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
; GFX10PLUS-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
; GFX10PLUS-SDAG: ; %bb.0:
; GFX10PLUS-SDAG-NEXT: v_sub_f16_e64 v0, s2, s3
diff --git a/llvm/test/CodeGen/AMDGPU/v_pack.ll b/llvm/test/CodeGen/AMDGPU/v_pack.ll
index c47947edd381c..257b519dcdbd3 100644
--- a/llvm/test/CodeGen/AMDGPU/v_pack.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_pack.ll
@@ -282,21 +282,18 @@ define amdgpu_kernel void @fptrunc(
; GISEL-LABEL: fptrunc:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GISEL-NEXT: s_mov_b32 s6, -1
-; GISEL-NEXT: s_mov_b32 s7, 0x31016000
-; GISEL-NEXT: s_mov_b32 s10, s6
-; GISEL-NEXT: s_mov_b32 s11, s7
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s8, s2
-; GISEL-NEXT: s_mov_b32 s9, s3
-; GISEL-NEXT: s_mov_b32 s4, s0
-; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; GISEL-NEXT: s_mov_b32 s5, s1
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GISEL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
+; GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GISEL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: s_mov_b32 s2, -1
+; GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GISEL-NEXT: s_endpgm
;
; GFX11-GCN-FAKE16-LABEL: fptrunc:
@@ -323,22 +320,19 @@ define amdgpu_kernel void @fptrunc(
; GFX11-GISEL-FAKE16-LABEL: fptrunc:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[0:1], off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX11-GCN-REAL16-LABEL: fptrunc:
@@ -363,20 +357,19 @@ define amdgpu_kernel void @fptrunc(
; GFX11-GISEL-REAL16-LABEL: fptrunc:
; GFX11-GISEL-REAL16: ; %bb.0:
; GFX11-GISEL-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-REAL16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-REAL16-NEXT: buffer_load_b64 v[0:1], off, s[8:11], 0
-; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-REAL16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-REAL16-NEXT: v_cvt_f16_f32_e32 v1.h, v1
-; GFX11-GISEL-REAL16-NEXT: v_cvt_f16_f32_e32 v1.l, v0
-; GFX11-GISEL-REAL16-NEXT: buffer_store_b32 v1, off, s[4:7], 0
+; GFX11-GISEL-REAL16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
+; GFX11-GISEL-REAL16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-REAL16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX11-GISEL-REAL16-NEXT: v_cvt_f16_f32_e32 v1.l, s3
+; GFX11-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-REAL16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-REAL16-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-GISEL-REAL16-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-REAL16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-REAL16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-REAL16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-REAL16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index c787b3d8025a9..5c2812ab3b5e0 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -251,61 +251,58 @@ define amdgpu_kernel void @basic_smax_smin_sgpr(ptr addrspace(1) %out, i32 inreg
; GISEL-GFX9-LABEL: basic_smax_smin_sgpr:
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0xff
-; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT: v_med3_i16 v2, s2, 0, v1
-; GISEL-GFX9-NEXT: v_med3_i16 v1, s3, 0, v1
-; GISEL-GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GISEL-GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
-; GISEL-GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s3, s3
+; GISEL-GFX9-NEXT: s_max_i32 s2, s2, 0
+; GISEL-GFX9-NEXT: s_max_i32 s3, s3, 0
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s3, s3
+; GISEL-GFX9-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-GFX9-NEXT: s_min_i32 s3, s3, 0xff
+; GISEL-GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX9-NEXT: s_endpgm
;
-; GISEL-GFX11-TRUE16-LABEL: basic_smax_smin_sgpr:
-; GISEL-GFX11-TRUE16: ; %bb.0:
-; GISEL-GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GISEL-GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX11-TRUE16-NEXT: v_med3_i16 v1.l, s2, 0, 0xff
-; GISEL-GFX11-TRUE16-NEXT: v_med3_i16 v1.h, s3, 0, 0xff
-; GISEL-GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GISEL-GFX11-TRUE16-NEXT: s_endpgm
-;
-; GISEL-GFX11-FAKE16-LABEL: basic_smax_smin_sgpr:
-; GISEL-GFX11-FAKE16: ; %bb.0:
-; GISEL-GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GISEL-GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GISEL-GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_med3_i16 v0, s2, 0, 0xff
-; GISEL-GFX11-FAKE16-NEXT: v_med3_i16 v1, s3, 0, 0xff
-; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GISEL-GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]
-; GISEL-GFX11-FAKE16-NEXT: s_endpgm
-;
-; GISEL-GFX12-TRUE16-LABEL: basic_smax_smin_sgpr:
-; GISEL-GFX12-TRUE16: ; %bb.0:
-; GISEL-GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GISEL-GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GISEL-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX12-TRUE16-NEXT: v_med3_i16 v1.l, s2, 0, 0xff
-; GISEL-GFX12-TRUE16-NEXT: v_med3_i16 v1.h, s3, 0, 0xff
-; GISEL-GFX12-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GISEL-GFX12-TRUE16-NEXT: s_endpgm
-;
-; GISEL-GFX12-FAKE16-LABEL: basic_smax_smin_sgpr:
-; GISEL-GFX12-FAKE16: ; %bb.0:
-; GISEL-GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GISEL-GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX12-FAKE16-NEXT: v_med3_i16 v0, s2, 0, 0xff
-; GISEL-GFX12-FAKE16-NEXT: v_med3_i16 v1, s3, 0, 0xff
-; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GISEL-GFX12-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]
-; GISEL-GFX12-FAKE16-NEXT: s_endpgm
+; GISEL-GFX11-LABEL: basic_smax_smin_sgpr:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s3, s3
+; GISEL-GFX11-NEXT: s_max_i32 s2, s2, 0
+; GISEL-GFX11-NEXT: s_max_i32 s3, s3, 0
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s3, s3
+; GISEL-GFX11-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-GFX11-NEXT: s_min_i32 s3, s3, 0xff
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX12-LABEL: basic_smax_smin_sgpr:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX12-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s3, s3
+; GISEL-GFX12-NEXT: s_max_i32 s2, s2, 0
+; GISEL-GFX12-NEXT: s_max_i32 s3, s3, 0
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s3, s3
+; GISEL-GFX12-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-GFX12-NEXT: s_min_i32 s3, s3, 0xff
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX12-NEXT: s_endpgm
%src0 = trunc i32 %src0ext to i16
%src1 = trunc i32 %src1ext to i16
@@ -704,18 +701,18 @@ define amdgpu_kernel void @vec_smax_smin_sgpr(ptr addrspace(1) %out, <2 x i16> i
; SDAG-GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; SDAG-GFX9-NEXT: s_endpgm
;
-; GFX11-LABEL: vec_smax_smin_sgpr:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_i16 v0, s2, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_i16 v0, 0xff, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: vec_smax_smin_sgpr:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_clause 0x1
+; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_pk_max_i16 v0, s2, 0
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT: v_pk_min_i16 v0, 0xff, v0 op_sel_hi:[0,1]
+; SDAG-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-GFX12-LABEL: vec_smax_smin_sgpr:
; SDAG-GFX12: ; %bb.0:
@@ -749,22 +746,69 @@ define amdgpu_kernel void @vec_smax_smin_sgpr(ptr addrspace(1) %out, <2 x i16> i
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s3, 0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT: v_pk_max_i16 v1, s2, 0
-; GISEL-GFX9-NEXT: s_movk_i32 s2, 0xff
-; GISEL-GFX9-NEXT: v_pk_min_i16 v1, v1, s2 op_sel_hi:[1,0]
-; GISEL-GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s4, s2
+; GISEL-GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GISEL-GFX9-NEXT: s_max_i32 s3, s4, s3
+; GISEL-GFX9-NEXT: s_max_i32 s2, s2, 0
+; GISEL-GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s3, s2
+; GISEL-GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s4, 0xff00ff
+; GISEL-GFX9-NEXT: s_min_i32 s3, s3, s4
+; GISEL-GFX9-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX9-NEXT: s_endpgm
;
+; GISEL-GFX11-LABEL: vec_smax_smin_sgpr:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s3, 0
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s4, s2
+; GISEL-GFX11-NEXT: s_ashr_i32 s2, s2, 16
+; GISEL-GFX11-NEXT: s_max_i32 s3, s4, s3
+; GISEL-GFX11-NEXT: s_max_i32 s2, s2, 0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s3, 0xff00ff
+; GISEL-GFX11-NEXT: s_sext_i32_i16 s4, s2
+; GISEL-GFX11-NEXT: s_ashr_i32 s2, s2, 16
+; GISEL-GFX11-NEXT: s_min_i32 s3, s4, s3
+; GISEL-GFX11-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
+;
; GISEL-GFX12-LABEL: vec_smax_smin_sgpr:
; GISEL-GFX12: ; %bb.0:
; GISEL-GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s3, 0
; GISEL-GFX12-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX12-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX12-NEXT: v_pk_max_i16 v0, s2, 0
-; GISEL-GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX12-NEXT: v_pk_min_i16 v0, 0xff, v0 op_sel_hi:[0,1]
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s4, s2
+; GISEL-GFX12-NEXT: s_ashr_i32 s2, s2, 16
+; GISEL-GFX12-NEXT: s_max_i32 s3, s4, s3
+; GISEL-GFX12-NEXT: s_max_i32 s2, s2, 0
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX12-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s3, 0xff00ff
+; GISEL-GFX12-NEXT: s_sext_i32_i16 s4, s2
+; GISEL-GFX12-NEXT: s_ashr_i32 s2, s2, 16
+; GISEL-GFX12-NEXT: s_min_i32 s3, s4, s3
+; GISEL-GFX12-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX12-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX12-NEXT: s_endpgm
@@ -1779,3 +1823,5 @@ define i16 @basic_smax_smin_vec_input_rev(<2 x i16> %src) {
ret i16 %cast
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index a4e6f1e8b6a2c..497d0e51b8742 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -1190,19 +1190,35 @@ define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
; GFX8-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v4i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v4i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v4i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v4i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v4i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v4i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v4i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1228,7 +1244,7 @@ define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1268,7 +1284,7 @@ define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -1277,42 +1293,97 @@ entry:
}
define i16 @test_vector_reduce_and_v8i16(<8 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v8i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v8i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_and_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v8i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v8i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v8i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v8i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v8i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v8i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v8i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v8i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1342,11 +1413,11 @@ define i16 @test_vector_reduce_and_v8i16(<8 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_and_v8i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1391,11 +1462,11 @@ define i16 @test_vector_reduce_and_v8i16(<8 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1405,58 +1476,145 @@ entry:
}
define i16 @test_vector_reduce_and_v16i16(<16 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v16i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v16i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_and_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v16i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v8, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v8, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v8, v4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v8, v6
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v8, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v16i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v16i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v16i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v16i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v16i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v16i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v16i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1496,16 +1654,16 @@ define i16 @test_vector_reduce_and_v16i16(<16 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_and_v16i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1560,16 +1718,16 @@ define i16 @test_vector_reduce_and_v16i16(<16 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1678,859 +1836,1582 @@ entry:
}
define i32 @test_vector_reduce_and_v4i32(<4 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v4i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_and_v4i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_and_v4i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_and_v4i32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_vector_reduce_and_v4i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: test_vector_reduce_and_v4i32:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %v)
- ret i32 %res
-}
-
-define i32 @test_vector_reduce_and_v8i32(<8 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v8i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v4i32:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_and_v8i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v4i32:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v8i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v4i32:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v8i32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v4i32:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_and_v8i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v4i32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_and_v8i32:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %v)
- ret i32 %res
-}
-
-define i32 @test_vector_reduce_and_v16i32(<16 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v16i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v4i32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_and_v16i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v4i32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v16i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v4i32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v16i32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v4i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_and_v16i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX11-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX11-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: test_vector_reduce_and_v4i32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_and_v16i32:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX12-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX12-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v4i32:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_and_v4i32:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
- %res = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %v)
+ %res = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %v)
ret i32 %res
}
-define i64 @test_vector_reduce_and_v2i64(<2 x i64> %v) {
-; GFX7-SDAG-LABEL: test_vector_reduce_and_v2i64:
+define i32 @test_vector_reduce_and_v8i32(<8 x i32> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v8i32:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-GISEL-LABEL: test_vector_reduce_and_v2i64:
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v8i32:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_and_v2i64:
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v8i32:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GISEL-LABEL: test_vector_reduce_and_v2i64:
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v8i32:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-SDAG-LABEL: test_vector_reduce_and_v2i64:
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v8i32:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: test_vector_reduce_and_v2i64:
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v8i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v2i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_vector_reduce_and_v2i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: test_vector_reduce_and_v2i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %v)
- ret i64 %res
-}
-
-define i64 @test_vector_reduce_and_v3i64(<3 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v3i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_and_v3i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_and_v3i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v8i32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v3i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v8i32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_and_v3i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v8i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_and_v8i32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_and_v3i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v8i32:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_and_v8i32:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
- %res = call i64 @llvm.vector.reduce.and.v3i64(<3 x i64> %v)
- ret i64 %res
+ %res = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %v)
+ ret i32 %res
}
-define i64 @test_vector_reduce_and_v4i64(<4 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v4i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+define i32 @test_vector_reduce_and_v16i32(<16 x i32> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v16i32:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_and_v4i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v16i32:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v4i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v16i32:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v4i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v16i32:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_and_v4i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v16i32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX9-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_and_v4i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v16i32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX9-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v16i32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX10-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX10-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v16i32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v16i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX11-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX11-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_and_v16i32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v16i32:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX12-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX12-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_and_v16i32:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %v)
+ ret i32 %res
+}
+
+define i64 @test_vector_reduce_and_v2i64(<2 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v2i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v2i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v2i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v2i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v2i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v2i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: test_vector_reduce_and_v2i64:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: test_vector_reduce_and_v2i64:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_vector_reduce_and_v2i64:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %v)
+ ret i64 %res
+}
+
+define i64 @test_vector_reduce_and_v3i64(<3 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v3i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v3i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v3i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v3i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v3i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v3i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: test_vector_reduce_and_v3i64:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: test_vector_reduce_and_v3i64:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_vector_reduce_and_v3i64:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i64 @llvm.vector.reduce.and.v3i64(<3 x i64> %v)
+ ret i64 %res
+}
+
+define i64 @test_vector_reduce_and_v4i64(<4 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v4i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v4i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v4i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v4i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v4i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v4i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v4i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v4i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v4i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_and_v4i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v4i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_and_v4i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %v)
+ ret i64 %res
+}
+
+define i64 @test_vector_reduce_and_v8i64(<8 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v8i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v8i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v8i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX8-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v8i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v8i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX9-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v8i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX9-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v8i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v8i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v8i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX11-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_and_v8i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v8i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX12-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_and_v8i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_and_b32_e32 v7, v7, v15
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
- %res = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %v)
+ %res = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %v)
ret i64 %res
}
-define i64 @test_vector_reduce_and_v8i64(<8 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v8i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX7-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_and_v8i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+define i64 @test_vector_reduce_and_v16i64(<16 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v16i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX7-SDAG-NEXT: buffer_load_dword v20, off, s[0:3], s32
+; GFX7-SDAG-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX7-SDAG-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX7-SDAG-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX7-SDAG-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX7-SDAG-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX7-SDAG-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX7-SDAG-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v15, v20
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v7, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, v3, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, v1, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v8i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v16i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX7-GISEL-NEXT: buffer_load_dword v16, off, s[0:3], s32
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX7-GISEL-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX7-GISEL-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX7-GISEL-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX7-GISEL-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX7-GISEL-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX7-GISEL-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v15, v16
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v7, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, v3, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v8i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX10-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v16i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX8-SDAG-NEXT: buffer_load_dword v20, off, s[0:3], s32
+; GFX8-SDAG-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX8-SDAG-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX8-SDAG-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX8-SDAG-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX8-SDAG-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX8-SDAG-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX8-SDAG-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX8-SDAG-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX8-SDAG-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v15, v20
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v7, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v2, v3, v2
+; GFX8-SDAG-NEXT: v_and_b32_e32 v1, v1, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_and_v8i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX11-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX11-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v16i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX8-GISEL-NEXT: buffer_load_dword v16, off, s[0:3], s32
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX8-GISEL-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX8-GISEL-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX8-GISEL-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX8-GISEL-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX8-GISEL-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX8-GISEL-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX8-GISEL-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX8-GISEL-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX8-GISEL-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX8-GISEL-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX8-GISEL-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v15, v16
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v7, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v3, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v2
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_and_v8i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX12-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX12-NEXT: v_and_b32_e32 v7, v7, v15
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %v)
- ret i64 %res
-}
-
-define i64 @test_vector_reduce_and_v16i64(<16 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v16i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, v4, v20
-; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX7-NEXT: v_and_b32_e32 v12, v12, v28
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v16
-; GFX7-NEXT: v_and_b32_e32 v8, v8, v24
-; GFX7-NEXT: v_and_b32_e32 v6, v6, v22
-; GFX7-NEXT: v_and_b32_e32 v14, v14, v30
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v18
-; GFX7-NEXT: v_and_b32_e32 v10, v10, v26
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_and_b32_e32 v5, v5, v21
-; GFX7-NEXT: v_and_b32_e32 v13, v13, v29
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v17
-; GFX7-NEXT: v_and_b32_e32 v9, v9, v25
-; GFX7-NEXT: v_and_b32_e32 v7, v7, v23
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v19
-; GFX7-NEXT: v_and_b32_e32 v11, v11, v27
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v2, v15, v20
-; GFX7-NEXT: v_and_b32_e32 v2, v7, v2
-; GFX7-NEXT: v_and_b32_e32 v2, v3, v2
-; GFX7-NEXT: v_and_b32_e32 v1, v1, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_and_v16i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32
+; GFX9-SDAG-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX9-SDAG-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX9-SDAG-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX9-SDAG-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX9-SDAG-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX9-SDAG-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX9-SDAG-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX9-SDAG-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX9-SDAG-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX9-SDAG-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v15, v31
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v7, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, v3, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, v1, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_and_v16i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v4, v4, v20
-; GFX8-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX8-NEXT: v_and_b32_e32 v12, v12, v28
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v16
-; GFX8-NEXT: v_and_b32_e32 v8, v8, v24
-; GFX8-NEXT: v_and_b32_e32 v6, v6, v22
-; GFX8-NEXT: v_and_b32_e32 v14, v14, v30
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v18
-; GFX8-NEXT: v_and_b32_e32 v10, v10, v26
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_and_b32_e32 v5, v5, v21
-; GFX8-NEXT: v_and_b32_e32 v13, v13, v29
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v17
-; GFX8-NEXT: v_and_b32_e32 v9, v9, v25
-; GFX8-NEXT: v_and_b32_e32 v7, v7, v23
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v19
-; GFX8-NEXT: v_and_b32_e32 v11, v11, v27
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v2, v15, v20
-; GFX8-NEXT: v_and_b32_e32 v2, v7, v2
-; GFX8-NEXT: v_and_b32_e32 v2, v3, v2
-; GFX8-NEXT: v_and_b32_e32 v1, v1, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_and_v16i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: scratch_load_dword v31, off, s32
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX9-GISEL-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX9-GISEL-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX9-GISEL-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX9-GISEL-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX9-GISEL-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX9-GISEL-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX9-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX9-GISEL-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX9-GISEL-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX9-GISEL-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX9-GISEL-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX9-GISEL-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX9-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v15, v31
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v7, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, v3, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_and_v16i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: scratch_load_dword v31, off, s32
-; GFX9-NEXT: v_and_b32_e32 v4, v4, v20
-; GFX9-NEXT: v_and_b32_e32 v12, v12, v28
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v16
-; GFX9-NEXT: v_and_b32_e32 v8, v8, v24
-; GFX9-NEXT: v_and_b32_e32 v6, v6, v22
-; GFX9-NEXT: v_and_b32_e32 v14, v14, v30
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v18
-; GFX9-NEXT: v_and_b32_e32 v10, v10, v26
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_and_b32_e32 v5, v5, v21
-; GFX9-NEXT: v_and_b32_e32 v13, v13, v29
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v17
-; GFX9-NEXT: v_and_b32_e32 v9, v9, v25
-; GFX9-NEXT: v_and_b32_e32 v7, v7, v23
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v19
-; GFX9-NEXT: v_and_b32_e32 v11, v11, v27
-; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, v15, v31
-; GFX9-NEXT: v_and_b32_e32 v2, v7, v2
-; GFX9-NEXT: v_and_b32_e32 v2, v3, v2
-; GFX9-NEXT: v_and_b32_e32 v1, v1, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: test_vector_reduce_and_v16i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX10-SDAG-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX10-SDAG-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX10-SDAG-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX10-SDAG-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX10-SDAG-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX10-SDAG-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX10-SDAG-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX10-SDAG-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX10-SDAG-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX10-SDAG-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX10-SDAG-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v10, v15, v31
+; GFX10-SDAG-NEXT: v_and_b32_e32 v7, v7, v10
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_and_v16i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v18
-; GFX10-NEXT: v_and_b32_e32 v10, v10, v26
-; GFX10-NEXT: v_and_b32_e32 v5, v5, v21
-; GFX10-NEXT: v_and_b32_e32 v4, v4, v20
-; GFX10-NEXT: v_and_b32_e32 v13, v13, v29
-; GFX10-NEXT: v_and_b32_e32 v12, v12, v28
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v17
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v16
-; GFX10-NEXT: v_and_b32_e32 v9, v9, v25
-; GFX10-NEXT: v_and_b32_e32 v8, v8, v24
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v19
-; GFX10-NEXT: v_and_b32_e32 v11, v11, v27
-; GFX10-NEXT: v_and_b32_e32 v7, v7, v23
-; GFX10-NEXT: v_and_b32_e32 v6, v6, v22
-; GFX10-NEXT: v_and_b32_e32 v14, v14, v30
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX10-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v10, v15, v31
-; GFX10-NEXT: v_and_b32_e32 v7, v7, v10
-; GFX10-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-GISEL-LABEL: test_vector_reduce_and_v16i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX10-GISEL-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX10-GISEL-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX10-GISEL-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX10-GISEL-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX10-GISEL-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX10-GISEL-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX10-GISEL-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX10-GISEL-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX10-GISEL-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v8, v15, v31
+; GFX10-GISEL-NEXT: v_and_b32_e32 v7, v7, v8
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_and_v16i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v18
-; GFX11-NEXT: v_and_b32_e32 v10, v10, v26
-; GFX11-NEXT: v_and_b32_e32 v5, v5, v21
-; GFX11-NEXT: v_and_b32_e32 v4, v4, v20
-; GFX11-NEXT: v_and_b32_e32 v13, v13, v29
-; GFX11-NEXT: v_and_b32_e32 v12, v12, v28
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v17
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v16
-; GFX11-NEXT: v_and_b32_e32 v9, v9, v25
-; GFX11-NEXT: v_and_b32_e32 v8, v8, v24
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v19
-; GFX11-NEXT: v_and_b32_e32 v11, v11, v27
-; GFX11-NEXT: v_and_b32_e32 v7, v7, v23
-; GFX11-NEXT: v_and_b32_e32 v6, v6, v22
-; GFX11-NEXT: v_and_b32_e32 v14, v14, v30
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX11-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v10, v15, v31
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v7, v7, v10
-; GFX11-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v16i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX11-SDAG-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX11-SDAG-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX11-SDAG-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX11-SDAG-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX11-SDAG-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX11-SDAG-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX11-SDAG-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX11-SDAG-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX11-SDAG-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX11-SDAG-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX11-SDAG-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX11-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v10, v15, v31
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v7, v7, v10
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_and_v16i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX11-GISEL-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX11-GISEL-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX11-GISEL-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX11-GISEL-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX11-GISEL-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX11-GISEL-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX11-GISEL-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX11-GISEL-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v8, v15, v31
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v7, v7, v8
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_and_v16i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v18
-; GFX12-NEXT: v_and_b32_e32 v10, v10, v26
-; GFX12-NEXT: v_and_b32_e32 v5, v5, v21
-; GFX12-NEXT: v_and_b32_e32 v4, v4, v20
-; GFX12-NEXT: v_and_b32_e32 v13, v13, v29
-; GFX12-NEXT: v_and_b32_e32 v12, v12, v28
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v17
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v16
-; GFX12-NEXT: v_and_b32_e32 v9, v9, v25
-; GFX12-NEXT: v_and_b32_e32 v8, v8, v24
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v19
-; GFX12-NEXT: v_and_b32_e32 v11, v11, v27
-; GFX12-NEXT: v_and_b32_e32 v7, v7, v23
-; GFX12-NEXT: v_and_b32_e32 v6, v6, v22
-; GFX12-NEXT: v_and_b32_e32 v14, v14, v30
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v10
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v11
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v8
-; GFX12-NEXT: v_and_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_and_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_and_b32_e32 v6, v6, v14
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v10, v15, v31
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v7, v7, v10
-; GFX12-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v16i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX12-SDAG-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX12-SDAG-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX12-SDAG-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX12-SDAG-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX12-SDAG-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX12-SDAG-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX12-SDAG-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX12-SDAG-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX12-SDAG-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX12-SDAG-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX12-SDAG-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX12-SDAG-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v10, v15, v31
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v7, v7, v10
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_and_v16i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v16
+; GFX12-GISEL-NEXT: v_and_b32_e32 v8, v8, v24
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v17
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v18
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v19
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, v4, v20
+; GFX12-GISEL-NEXT: v_and_b32_e32 v5, v5, v21
+; GFX12-GISEL-NEXT: v_and_b32_e32 v6, v6, v22
+; GFX12-GISEL-NEXT: v_and_b32_e32 v7, v7, v23
+; GFX12-GISEL-NEXT: v_and_b32_e32 v9, v9, v25
+; GFX12-GISEL-NEXT: v_and_b32_e32 v10, v10, v26
+; GFX12-GISEL-NEXT: v_and_b32_e32 v11, v11, v27
+; GFX12-GISEL-NEXT: v_and_b32_e32 v12, v12, v28
+; GFX12-GISEL-NEXT: v_and_b32_e32 v13, v13, v29
+; GFX12-GISEL-NEXT: v_and_b32_e32 v14, v14, v30
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v8
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v9
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v10
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v11
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_and_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v8, v15, v31
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v7, v7, v8
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> %v)
ret i64 %res
@@ -2561,6 +3442,4 @@ declare i64 @llvm.vector.reduce.and.v8i64(<8 x i64>)
declare i64 @llvm.vector.reduce.and.v16i64(<16 x i64>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX11-GISEL-FAKE16: {{.*}}
-; GFX11-SDAG: {{.*}}
; GFX12-GISEL-FAKE16: {{.*}}
-; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
index a7a10aacf8bed..7e6bbacea862a 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
@@ -2474,19 +2474,34 @@ define i64 @test_vector_reduce_mul_v2i64(<2 x i64> %v) {
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v6, v5, v[7:8]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_mul_v2i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u32 v4, v1, v2
-; GFX12-NEXT: v_mul_lo_u32 v3, v0, v3
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v1, v1, v3, v4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_mul_v2i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v3, v1, v4
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> %v)
ret i64 %res
@@ -2628,24 +2643,45 @@ define i64 @test_vector_reduce_mul_v3i64(<3 x i64> %v) {
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v10, v4, v[8:9]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_mul_v3i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u32 v6, v1, v2
-; GFX12-NEXT: v_mul_lo_u32 v3, v0, v3
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_add3_u32 v1, v1, v3, v6
-; GFX12-NEXT: v_mul_lo_u32 v2, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mul_lo_u32 v3, v1, v4
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v4, 0
-; GFX12-NEXT: v_add3_u32 v1, v1, v2, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_mul_v3i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v1, v2
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v3, v6
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v2, v0, v5
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v1, v4
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v4, 0
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v2, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v6, v0, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v3, v1, v6
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v2, v0, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v0, v5
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v3, v1, v2
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v3i64(<3 x i64> %v)
ret i64 %res
@@ -2826,29 +2862,56 @@ define i64 @test_vector_reduce_mul_v4i64(<4 x i64> %v) {
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v5, v9, v[3:4]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_mul_v4i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u32 v8, v3, v6
-; GFX12-NEXT: v_mul_lo_u32 v7, v2, v7
-; GFX12-NEXT: v_mul_lo_u32 v9, v1, v4
-; GFX12-NEXT: v_mul_lo_u32 v5, v0, v5
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v4, 0
-; GFX12-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v6, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_add3_u32 v1, v1, v5, v9
-; GFX12-NEXT: v_add3_u32 v3, v3, v7, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_mul_lo_u32 v4, v1, v2
-; GFX12-NEXT: v_mul_lo_u32 v3, v0, v3
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v1, v1, v3, v4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_mul_v4i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v8, v3, v6
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v7, v2, v7
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v1, v4
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v4, 0
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v6, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v5, v9
+; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v7, v8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v8, v0, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v0, v5
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v9, v2, v6
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v7, v2, v7
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v6
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_add3_u32 v3, v7, v3, v9
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v5, v1, v8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v3, v1, v4
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> %v)
ret i64 %res
@@ -3185,49 +3248,96 @@ define i64 @test_vector_reduce_mul_v8i64(<8 x i64> %v) {
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v10, v24, v[3:4]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_mul_v8i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mul_lo_u32 v16, v5, v12
-; GFX12-NEXT: v_mul_lo_u32 v13, v4, v13
-; GFX12-NEXT: v_mul_lo_u32 v17, v1, v8
-; GFX12-NEXT: v_mul_lo_u32 v9, v0, v9
-; GFX12-NEXT: v_mul_lo_u32 v18, v7, v14
-; GFX12-NEXT: v_mul_lo_u32 v15, v6, v15
-; GFX12-NEXT: v_mul_lo_u32 v19, v3, v10
-; GFX12-NEXT: v_mul_lo_u32 v11, v2, v11
-; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], null, v2, v10, 0
-; GFX12-NEXT: v_mad_co_u64_u32 v[5:6], null, v6, v14, 0
-; GFX12-NEXT: v_mad_co_u64_u32 v[7:8], null, v0, v8, 0
-; GFX12-NEXT: v_mad_co_u64_u32 v[3:4], null, v4, v12, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_add3_u32 v0, v2, v11, v19
-; GFX12-NEXT: v_add3_u32 v2, v6, v15, v18
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_add3_u32 v6, v8, v9, v17
-; GFX12-NEXT: v_add3_u32 v4, v4, v13, v16
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_mul_lo_u32 v8, v0, v5
-; GFX12-NEXT: v_mul_lo_u32 v9, v1, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_mul_lo_u32 v6, v6, v3
-; GFX12-NEXT: v_mul_lo_u32 v4, v7, v4
-; GFX12-NEXT: v_mad_co_u64_u32 v[2:3], null, v7, v3, 0
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v1, v5, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_add3_u32 v3, v3, v4, v6
-; GFX12-NEXT: v_add3_u32 v1, v1, v9, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_mul_lo_u32 v3, v3, v0
-; GFX12-NEXT: v_mul_lo_u32 v4, v2, v1
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, v0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v1, v1, v4, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_mul_v8i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v16, v5, v12
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v13, v4, v13
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v17, v1, v8
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v18, v7, v14
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v15, v6, v15
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v19, v3, v10
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v11, v2, v11
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[1:2], null, v2, v10, 0
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[5:6], null, v6, v14, 0
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[7:8], null, v0, v8, 0
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[3:4], null, v4, v12, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_add3_u32 v0, v2, v11, v19
+; GFX12-SDAG-NEXT: v_add3_u32 v2, v6, v15, v18
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_add3_u32 v6, v8, v9, v17
+; GFX12-SDAG-NEXT: v_add3_u32 v4, v4, v13, v16
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v8, v0, v5
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v1, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v6, v3
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v7, v4
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v7, v3, 0
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v1, v5, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v4, v6
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v9, v8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v2, v1
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, v0, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v4, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v16, v0, v8
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v17, v0, v8
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v9
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v8
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v8, v2, v10
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v9, v2, v11
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v11, v4, v12
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v13, v4, v13
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v5, v12
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v18, v6, v14
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v15, v6, v15
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v7, v7, v14
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v10
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_add3_u32 v5, v13, v5, v11
+; GFX12-GISEL-NEXT: v_add3_u32 v0, v0, v1, v17
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v15, v7, v18
+; GFX12-GISEL-NEXT: v_add3_u32 v3, v9, v3, v8
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v7, v16, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v16, v5
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v8, v2, v6
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v6
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v4, v16, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v1, v3, v8
+; GFX12-GISEL-NEXT: v_add3_u32 v0, v5, v0, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v3, v4, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v0, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v4, v2
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %v)
ret i64 %res
@@ -3920,83 +4030,176 @@ define i64 @test_vector_reduce_mul_v16i64(<16 x i64> %v) {
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v6, v3, v[9:10]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_mul_v16i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-NEXT: v_mul_lo_u32 v32, v11, v26
-; GFX12-NEXT: v_mul_lo_u32 v27, v10, v27
-; GFX12-NEXT: v_mad_co_u64_u32 v[10:11], null, v10, v26, 0
-; GFX12-NEXT: v_mul_lo_u32 v26, v3, v18
-; GFX12-NEXT: v_mul_lo_u32 v19, v2, v19
-; GFX12-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v18, 0
-; GFX12-NEXT: v_mul_lo_u32 v18, v7, v22
-; GFX12-NEXT: v_mul_lo_u32 v23, v6, v23
-; GFX12-NEXT: v_mad_co_u64_u32 v[6:7], null, v6, v22, 0
-; GFX12-NEXT: v_mul_lo_u32 v22, v9, v24
-; GFX12-NEXT: v_mul_lo_u32 v25, v8, v25
-; GFX12-NEXT: v_mad_co_u64_u32 v[8:9], null, v8, v24, 0
-; GFX12-NEXT: v_mul_lo_u32 v24, v1, v16
-; GFX12-NEXT: v_mul_lo_u32 v17, v0, v17
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v16, 0
-; GFX12-NEXT: v_mul_lo_u32 v33, v15, v30
-; GFX12-NEXT: v_mul_lo_u32 v34, v13, v28
-; GFX12-NEXT: v_mul_lo_u32 v29, v12, v29
-; GFX12-NEXT: v_mad_co_u64_u32 v[12:13], null, v12, v28, 0
-; GFX12-NEXT: v_mul_lo_u32 v28, v5, v20
-; GFX12-NEXT: v_mul_lo_u32 v21, v4, v21
-; GFX12-NEXT: v_mad_co_u64_u32 v[4:5], null, v4, v20, 0
-; GFX12-NEXT: v_mad_co_u64_u32 v[15:16], null, v14, v30, 0
-; GFX12-NEXT: v_add3_u32 v1, v1, v17, v24
-; GFX12-NEXT: v_add3_u32 v11, v11, v27, v32
-; GFX12-NEXT: v_add3_u32 v3, v3, v19, v26
-; GFX12-NEXT: v_add3_u32 v7, v7, v23, v18
-; GFX12-NEXT: v_add3_u32 v9, v9, v25, v22
-; GFX12-NEXT: v_add3_u32 v23, v13, v29, v34
-; GFX12-NEXT: v_add3_u32 v5, v5, v21, v28
-; GFX12-NEXT: v_mad_co_u64_u32 v[17:18], null, v0, v8, 0
-; GFX12-NEXT: v_mul_lo_u32 v8, v1, v8
-; GFX12-NEXT: v_mad_co_u64_u32 v[19:20], null, v6, v15, 0
-; GFX12-NEXT: v_mad_co_u64_u32 v[21:22], null, v2, v10, 0
-; GFX12-NEXT: v_mul_lo_u32 v5, v5, v12
-; GFX12-NEXT: v_mul_lo_u32 v3, v3, v10
-; GFX12-NEXT: v_mul_lo_u32 v2, v2, v11
-; GFX12-NEXT: v_mul_lo_u32 v9, v0, v9
-; GFX12-NEXT: v_mul_lo_u32 v7, v7, v15
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_add3_u32 v10, v22, v2, v3
-; GFX12-NEXT: v_add3_u32 v8, v18, v9, v8
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mul_lo_u32 v24, v14, v31
-; GFX12-NEXT: v_mad_co_u64_u32 v[13:14], null, v4, v12, 0
-; GFX12-NEXT: v_mul_lo_u32 v4, v4, v23
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_add3_u32 v1, v16, v24, v33
-; GFX12-NEXT: v_mad_co_u64_u32 v[2:3], null, v17, v13, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_add3_u32 v4, v14, v4, v5
-; GFX12-NEXT: v_mul_lo_u32 v6, v6, v1
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v21, v19, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_mul_lo_u32 v4, v17, v4
-; GFX12-NEXT: v_add3_u32 v5, v20, v6, v7
-; GFX12-NEXT: v_mul_lo_u32 v6, v10, v19
-; GFX12-NEXT: v_mul_lo_u32 v7, v8, v13
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_mul_lo_u32 v5, v21, v5
-; GFX12-NEXT: v_add3_u32 v3, v3, v4, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_add3_u32 v1, v1, v5, v6
-; GFX12-NEXT: v_mul_lo_u32 v3, v3, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mul_lo_u32 v4, v2, v1
-; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, v0, 0
-; GFX12-NEXT: v_add3_u32 v1, v1, v4, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_mul_v16i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v32, v11, v26
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v27, v10, v27
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[10:11], null, v10, v26, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v26, v3, v18
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v19, v2, v19
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v18, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v18, v7, v22
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v23, v6, v23
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[6:7], null, v6, v22, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v22, v9, v24
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v25, v8, v25
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[8:9], null, v8, v24, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v24, v1, v16
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v17, v0, v17
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v16, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v33, v15, v30
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v34, v13, v28
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v29, v12, v29
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[12:13], null, v12, v28, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v28, v5, v20
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v21, v4, v21
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[4:5], null, v4, v20, 0
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[15:16], null, v14, v30, 0
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v17, v24
+; GFX12-SDAG-NEXT: v_add3_u32 v11, v11, v27, v32
+; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v19, v26
+; GFX12-SDAG-NEXT: v_add3_u32 v7, v7, v23, v18
+; GFX12-SDAG-NEXT: v_add3_u32 v9, v9, v25, v22
+; GFX12-SDAG-NEXT: v_add3_u32 v23, v13, v29, v34
+; GFX12-SDAG-NEXT: v_add3_u32 v5, v5, v21, v28
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[17:18], null, v0, v8, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v8, v1, v8
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[19:20], null, v6, v15, 0
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[21:22], null, v2, v10, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v5, v5, v12
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v10
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v2, v2, v11
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_add3_u32 v10, v22, v2, v3
+; GFX12-SDAG-NEXT: v_add3_u32 v8, v18, v9, v8
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v24, v14, v31
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[13:14], null, v4, v12, 0
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v4, v23
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v16, v24, v33
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v17, v13, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_add3_u32 v4, v14, v4, v5
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v6, v1
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v21, v19, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v17, v4
+; GFX12-SDAG-NEXT: v_add3_u32 v5, v20, v6, v7
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v10, v19
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v7, v8, v13
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v5, v21, v5
+; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v4, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v5, v6
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v2, v1
+; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, v0, 0
+; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v4, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v32, v0, v16
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v33, v0, v16
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v17
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v16
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v16, v2, v18
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v17, v2, v18
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v19
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v18
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v18, v4, v20
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v19, v4, v20
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v4, v4, v21
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v5, v20
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v20, v6, v22
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v21, v6, v22
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v6, v6, v23
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v7, v7, v22
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v22, v8, v24
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v23, v8, v24
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v8, v8, v25
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v9, v9, v24
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v24, v10, v26
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v25, v10, v26
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v10, v10, v27
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v11, v11, v26
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v26, v12, v28
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v27, v12, v28
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v12, v12, v29
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v13, v13, v28
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v29, v14, v30
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v15, v15, v30
+; GFX12-GISEL-NEXT: v_add3_u32 v0, v0, v1, v33
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v2, v3, v17
+; GFX12-GISEL-NEXT: v_add3_u32 v2, v4, v5, v19
+; GFX12-GISEL-NEXT: v_add3_u32 v4, v8, v9, v23
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v28, v14, v30
+; GFX12-GISEL-NEXT: v_add3_u32 v3, v6, v7, v21
+; GFX12-GISEL-NEXT: v_add3_u32 v5, v10, v11, v25
+; GFX12-GISEL-NEXT: v_add3_u32 v6, v12, v13, v27
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v8, v32, v22
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v11, v16, v24
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v13, v18, v26
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v4, v32, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v16, v5
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v24
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v6, v18, v6
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v26
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v28
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v22
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v7, v32, v22
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v10, v16, v24
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v12, v18, v26
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v5, v1, v11
+; GFX12-GISEL-NEXT: v_add3_u32 v2, v6, v2, v13
+; GFX12-GISEL-NEXT: v_add3_u32 v0, v4, v0, v8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v16, v7, v12
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v7, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v12
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v7, v12
+; GFX12-GISEL-NEXT: v_add3_u32 v0, v2, v0, v16
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v9, v14, v31
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v14, v20, v28
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add3_u32 v9, v9, v15, v29
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v15, v20, v28
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v9, v20, v9
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v4, v10, v15
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v15
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v6, v10, v15
+; GFX12-GISEL-NEXT: v_add3_u32 v3, v9, v3, v14
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v10, v3
+; GFX12-GISEL-NEXT: v_mul_hi_u32 v2, v5, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v3, v1, v4
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v0, v6
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v5, v6
+; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v5, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add3_u32 v1, v1, v3, v2
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v16i64(<16 x i64> %v)
ret i64 %res
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index c3aa0d752e4cb..e557d8fab4c78 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -1014,19 +1014,35 @@ define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v4i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v4i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v4i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v4i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v4i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v4i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v4i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1052,7 +1068,7 @@ define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1092,7 +1108,7 @@ define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -1101,41 +1117,96 @@ entry:
}
define i16 @test_vector_reduce_or_v8i16(<8 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v8i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v8i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v8i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v8i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v8i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v8i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v8i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v8i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v8i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v8i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1163,10 +1234,11 @@ define i16 @test_vector_reduce_or_v8i16(<8 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_or_v8i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1209,10 +1281,11 @@ define i16 @test_vector_reduce_or_v8i16(<8 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1222,55 +1295,142 @@ entry:
}
define i16 @test_vector_reduce_or_v16i16(<16 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v16i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v16i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v16i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v8, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v8, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v8, v4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v8, v6
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v8, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v16i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX9-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v16i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX10-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v16i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v16i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v16i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v16i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v16i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1304,14 +1464,17 @@ define i16 @test_vector_reduce_or_v16i16(<16 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_or_v16i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1359,14 +1522,17 @@ define i16 @test_vector_reduce_or_v16i16(<16 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -1468,25 +1634,41 @@ entry:
}
define i32 @test_vector_reduce_or_v4i32(<4 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v4i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_or_v4i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v4i32:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v4i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v4i32:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v4i32:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v4i32:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: test_vector_reduce_or_v4i32:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1523,259 +1705,437 @@ entry:
}
define i32 @test_vector_reduce_or_v8i32(<8 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v8i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_or_v8i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v8i32:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v8i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX9-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v8i32:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v8i32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX10-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v8i32:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_or_v8i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX11-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v8i32:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_or_v8i32:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX12-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX12-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %v)
- ret i32 %res
-}
-
-define i32 @test_vector_reduce_or_v16i32(<16 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v16i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v8i32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v16i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v8i32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v16i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX9-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX9-NEXT: v_or3_b32 v0, v0, v8, v4
-; GFX9-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX9-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v8i32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v16i32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX10-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX10-NEXT: v_or3_b32 v0, v0, v8, v4
-; GFX10-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX10-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v8i32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_or_v16i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX11-NEXT: v_or3_b32 v0, v0, v8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX11-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_or_v8i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_or_v8i32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_or_v16i32:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX12-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX12-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX12-NEXT: v_or3_b32 v0, v0, v8, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX12-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_or_v8i32:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_or_v8i32:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
- %res = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %v)
+ %res = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %v)
ret i32 %res
}
-define i64 @test_vector_reduce_or_v2i64(<2 x i64> %v) {
-; GFX7-SDAG-LABEL: test_vector_reduce_or_v2i64:
+define i32 @test_vector_reduce_or_v16i32(<16 x i32> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v16i32:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-GISEL-LABEL: test_vector_reduce_or_v2i64:
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v16i32:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_or_v2i64:
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v16i32:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GISEL-LABEL: test_vector_reduce_or_v2i64:
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v16i32:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-SDAG-LABEL: test_vector_reduce_or_v2i64:
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v16i32:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX9-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX9-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX9-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: test_vector_reduce_or_v2i64:
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v16i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v2i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v16i32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX10-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX10-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX10-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_or_v2i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v16i32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX10-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_or_v2i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-SDAG-LABEL: test_vector_reduce_or_v16i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX11-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_or_v16i32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX11-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX11-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX11-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_or_v16i32:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX12-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_or_v16i32:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX12-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX12-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %v)
+ ret i32 %res
+}
+
+define i64 @test_vector_reduce_or_v2i64(<2 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v2i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v2i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v2i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v2i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v2i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v2i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: test_vector_reduce_or_v2i64:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: test_vector_reduce_or_v2i64:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_vector_reduce_or_v2i64:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX12-NEXT: v_or_b32_e32 v1, v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> %v)
@@ -1783,23 +2143,41 @@ entry:
}
define i64 @test_vector_reduce_or_v3i64(<3 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v3i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v3i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v3i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v3i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v3i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v3i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_or_v3i64:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1845,36 +2223,67 @@ entry:
}
define i64 @test_vector_reduce_or_v4i64(<4 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v4i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v4i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v4i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v4i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v4i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX9-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v4i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v4i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v4i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v4i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_or_v4i64:
; GFX10: ; %bb.0: ; %entry
@@ -1914,304 +2323,598 @@ entry:
}
define i64 @test_vector_reduce_or_v8i64(<8 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v8i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v8i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v8i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v8i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v8i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX9-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX9-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX9-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v8i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX8-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v8i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX10-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX10-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX10-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX10-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v8i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_or_v8i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX11-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX11-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX11-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v8i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX9-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX9-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX9-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_or_v8i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX12-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX12-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX12-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX12-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX12-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v8i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX9-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v8i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX10-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX10-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v8i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX10-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_or_v8i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX11-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX11-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX11-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX11-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_or_v8i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX11-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX11-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX11-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_or_v8i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX12-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX12-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX12-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX12-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_or_v8i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX12-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX12-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %v)
ret i64 %res
}
define i64 @test_vector_reduce_or_v16i64(<16 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v16i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v20
-; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX7-NEXT: v_or_b32_e32 v12, v12, v28
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v16
-; GFX7-NEXT: v_or_b32_e32 v8, v8, v24
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v22
-; GFX7-NEXT: v_or_b32_e32 v14, v14, v30
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v18
-; GFX7-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v21
-; GFX7-NEXT: v_or_b32_e32 v13, v13, v29
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v17
-; GFX7-NEXT: v_or_b32_e32 v9, v9, v25
-; GFX7-NEXT: v_or_b32_e32 v7, v7, v23
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v19
-; GFX7-NEXT: v_or_b32_e32 v11, v11, v27
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v2, v15, v20
-; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v16i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v4, v4, v20
+; GFX7-SDAG-NEXT: buffer_load_dword v20, off, s[0:3], s32
+; GFX7-SDAG-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v16
+; GFX7-SDAG-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX7-SDAG-NEXT: v_or_b32_e32 v6, v6, v22
+; GFX7-SDAG-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX7-SDAG-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v5, v5, v21
+; GFX7-SDAG-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v17
+; GFX7-SDAG-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX7-SDAG-NEXT: v_or_b32_e32 v7, v7, v23
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX7-SDAG-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v15, v20
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v16i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v20
-; GFX8-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX8-NEXT: v_or_b32_e32 v12, v12, v28
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v16
-; GFX8-NEXT: v_or_b32_e32 v8, v8, v24
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v22
-; GFX8-NEXT: v_or_b32_e32 v14, v14, v30
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v18
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v21
-; GFX8-NEXT: v_or_b32_e32 v13, v13, v29
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v17
-; GFX8-NEXT: v_or_b32_e32 v9, v9, v25
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v23
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v19
-; GFX8-NEXT: v_or_b32_e32 v11, v11, v27
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v15, v20
-; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v16i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v16
+; GFX7-GISEL-NEXT: buffer_load_dword v16, off, s[0:3], s32
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v4, v20
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v6, v22
+; GFX7-GISEL-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX7-GISEL-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX7-GISEL-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX7-GISEL-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v17
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v5, v21
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v7, v23
+; GFX7-GISEL-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX7-GISEL-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX7-GISEL-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v15, v16
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v16i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: scratch_load_dword v31, off, s32
-; GFX9-NEXT: v_or_b32_e32 v14, v14, v30
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v18
-; GFX9-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX9-NEXT: v_or_b32_e32 v12, v12, v28
-; GFX9-NEXT: v_or_b32_e32 v8, v8, v24
-; GFX9-NEXT: v_or3_b32 v6, v6, v22, v14
-; GFX9-NEXT: v_or3_b32 v0, v0, v16, v8
-; GFX9-NEXT: v_or3_b32 v4, v4, v20, v12
-; GFX9-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX9-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v19
-; GFX9-NEXT: v_or_b32_e32 v11, v11, v27
-; GFX9-NEXT: v_or_b32_e32 v13, v13, v29
-; GFX9-NEXT: v_or_b32_e32 v9, v9, v25
-; GFX9-NEXT: v_or3_b32 v1, v1, v17, v9
-; GFX9-NEXT: v_or3_b32 v5, v5, v21, v13
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v2, v15, v31
-; GFX9-NEXT: v_or3_b32 v2, v7, v23, v2
-; GFX9-NEXT: v_or3_b32 v2, v3, v11, v2
-; GFX9-NEXT: v_or3_b32 v1, v1, v5, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v16i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v4, v4, v20
+; GFX8-SDAG-NEXT: buffer_load_dword v20, off, s[0:3], s32
+; GFX8-SDAG-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v16
+; GFX8-SDAG-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX8-SDAG-NEXT: v_or_b32_e32 v6, v6, v22
+; GFX8-SDAG-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX8-SDAG-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v5, v5, v21
+; GFX8-SDAG-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v17
+; GFX8-SDAG-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX8-SDAG-NEXT: v_or_b32_e32 v7, v7, v23
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX8-SDAG-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v15, v20
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v16i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT: v_or_b32_e32 v14, v14, v30
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v19
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v18
-; GFX10-NEXT: v_or_b32_e32 v11, v11, v27
-; GFX10-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX10-NEXT: v_or_b32_e32 v9, v9, v25
-; GFX10-NEXT: v_or_b32_e32 v13, v13, v29
-; GFX10-NEXT: v_or_b32_e32 v8, v8, v24
-; GFX10-NEXT: v_or_b32_e32 v12, v12, v28
-; GFX10-NEXT: v_or3_b32 v6, v6, v22, v14
-; GFX10-NEXT: v_or3_b32 v1, v1, v17, v9
-; GFX10-NEXT: v_or3_b32 v5, v5, v21, v13
-; GFX10-NEXT: v_or3_b32 v0, v0, v16, v8
-; GFX10-NEXT: v_or3_b32 v4, v4, v20, v12
-; GFX10-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX10-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v15, v15, v31
-; GFX10-NEXT: v_or3_b32 v7, v7, v23, v15
-; GFX10-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX10-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v16i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v16
+; GFX8-GISEL-NEXT: buffer_load_dword v16, off, s[0:3], s32
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v20
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v22
+; GFX8-GISEL-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX8-GISEL-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX8-GISEL-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX8-GISEL-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v17
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX8-GISEL-NEXT: v_or_b32_e32 v5, v5, v21
+; GFX8-GISEL-NEXT: v_or_b32_e32 v7, v7, v23
+; GFX8-GISEL-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX8-GISEL-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX8-GISEL-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v15, v16
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_or_v16i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: v_or_b32_e32 v14, v14, v30
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v19
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v18
-; GFX11-NEXT: v_or_b32_e32 v11, v11, v27
-; GFX11-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX11-NEXT: v_or_b32_e32 v9, v9, v25
-; GFX11-NEXT: v_or_b32_e32 v13, v13, v29
-; GFX11-NEXT: v_or_b32_e32 v8, v8, v24
-; GFX11-NEXT: v_or_b32_e32 v12, v12, v28
-; GFX11-NEXT: v_or3_b32 v6, v6, v22, v14
-; GFX11-NEXT: v_or3_b32 v1, v1, v17, v9
-; GFX11-NEXT: v_or3_b32 v5, v5, v21, v13
-; GFX11-NEXT: v_or3_b32 v0, v0, v16, v8
-; GFX11-NEXT: v_or3_b32 v4, v4, v20, v12
-; GFX11-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_or_b32_e32 v15, v15, v31
-; GFX11-NEXT: v_or3_b32 v7, v7, v23, v15
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX11-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v16i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32
+; GFX9-SDAG-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX9-SDAG-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX9-SDAG-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX9-SDAG-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX9-SDAG-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX9-SDAG-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX9-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX9-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX9-SDAG-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX9-SDAG-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX9-SDAG-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX9-SDAG-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX9-SDAG-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v15, v31
+; GFX9-SDAG-NEXT: v_or3_b32 v2, v7, v23, v2
+; GFX9-SDAG-NEXT: v_or3_b32 v2, v3, v11, v2
+; GFX9-SDAG-NEXT: v_or3_b32 v1, v1, v5, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_or_v16i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-NEXT: v_or_b32_e32 v14, v14, v30
-; GFX12-NEXT: v_or_b32_e32 v3, v3, v19
-; GFX12-NEXT: v_or_b32_e32 v2, v2, v18
-; GFX12-NEXT: v_or_b32_e32 v11, v11, v27
-; GFX12-NEXT: v_or_b32_e32 v10, v10, v26
-; GFX12-NEXT: v_or_b32_e32 v9, v9, v25
-; GFX12-NEXT: v_or_b32_e32 v13, v13, v29
-; GFX12-NEXT: v_or_b32_e32 v8, v8, v24
-; GFX12-NEXT: v_or_b32_e32 v12, v12, v28
-; GFX12-NEXT: v_or3_b32 v6, v6, v22, v14
-; GFX12-NEXT: v_or3_b32 v1, v1, v17, v9
-; GFX12-NEXT: v_or3_b32 v5, v5, v21, v13
-; GFX12-NEXT: v_or3_b32 v0, v0, v16, v8
-; GFX12-NEXT: v_or3_b32 v4, v4, v20, v12
-; GFX12-NEXT: v_or3_b32 v2, v2, v10, v6
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_or3_b32 v0, v0, v4, v2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_or_b32_e32 v15, v15, v31
-; GFX12-NEXT: v_or3_b32 v7, v7, v23, v15
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_or3_b32 v3, v3, v11, v7
-; GFX12-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v16i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: scratch_load_dword v31, off, s32
+; GFX9-GISEL-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX9-GISEL-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX9-GISEL-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX9-GISEL-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX9-GISEL-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX9-GISEL-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX9-GISEL-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX9-GISEL-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX9-GISEL-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX9-GISEL-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX9-GISEL-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, v15, v31
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v7, v23, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v3, v11, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v5, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v16i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-SDAG-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX10-SDAG-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX10-SDAG-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX10-SDAG-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX10-SDAG-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX10-SDAG-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX10-SDAG-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX10-SDAG-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX10-SDAG-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX10-SDAG-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX10-SDAG-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX10-SDAG-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX10-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX10-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v15, v15, v31
+; GFX10-SDAG-NEXT: v_or3_b32 v7, v7, v23, v15
+; GFX10-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX10-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v16i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-GISEL-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX10-GISEL-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX10-GISEL-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX10-GISEL-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX10-GISEL-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX10-GISEL-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX10-GISEL-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX10-GISEL-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX10-GISEL-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX10-GISEL-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX10-GISEL-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX10-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v15, v15, v31
+; GFX10-GISEL-NEXT: v_or3_b32 v7, v7, v23, v15
+; GFX10-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_or_v16i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-SDAG-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX11-SDAG-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX11-SDAG-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX11-SDAG-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX11-SDAG-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX11-SDAG-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX11-SDAG-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX11-SDAG-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX11-SDAG-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX11-SDAG-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX11-SDAG-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX11-SDAG-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX11-SDAG-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX11-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v15, v15, v31
+; GFX11-SDAG-NEXT: v_or3_b32 v7, v7, v23, v15
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX11-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_or_v16i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-GISEL-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX11-GISEL-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX11-GISEL-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX11-GISEL-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX11-GISEL-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX11-GISEL-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX11-GISEL-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX11-GISEL-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX11-GISEL-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX11-GISEL-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX11-GISEL-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX11-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_or_b32_e32 v15, v15, v31
+; GFX11-GISEL-NEXT: v_or3_b32 v7, v7, v23, v15
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_or_v16i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-SDAG-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX12-SDAG-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX12-SDAG-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX12-SDAG-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX12-SDAG-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX12-SDAG-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX12-SDAG-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX12-SDAG-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX12-SDAG-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX12-SDAG-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX12-SDAG-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX12-SDAG-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX12-SDAG-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX12-SDAG-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v15, v15, v31
+; GFX12-SDAG-NEXT: v_or3_b32 v7, v7, v23, v15
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX12-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_or_v16i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-GISEL-NEXT: v_or_b32_e32 v14, v14, v30
+; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX12-GISEL-NEXT: v_or_b32_e32 v3, v3, v19
+; GFX12-GISEL-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX12-GISEL-NEXT: v_or_b32_e32 v9, v9, v25
+; GFX12-GISEL-NEXT: v_or_b32_e32 v10, v10, v26
+; GFX12-GISEL-NEXT: v_or_b32_e32 v11, v11, v27
+; GFX12-GISEL-NEXT: v_or_b32_e32 v12, v12, v28
+; GFX12-GISEL-NEXT: v_or_b32_e32 v13, v13, v29
+; GFX12-GISEL-NEXT: v_or3_b32 v6, v6, v22, v14
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v16, v8
+; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v17, v9
+; GFX12-GISEL-NEXT: v_or3_b32 v4, v4, v20, v12
+; GFX12-GISEL-NEXT: v_or3_b32 v5, v5, v21, v13
+; GFX12-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v15, v15, v31
+; GFX12-GISEL-NEXT: v_or3_b32 v7, v7, v23, v15
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> %v)
ret i64 %res
@@ -2242,6 +2945,4 @@ declare i64 @llvm.vector.reduce.or.v8i64(<8 x i64>)
declare i64 @llvm.vector.reduce.or.v16i64(<16 x i64>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX11-GISEL-FAKE16: {{.*}}
-; GFX11-SDAG: {{.*}}
; GFX12-GISEL-FAKE16: {{.*}}
-; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
index 9ae6a15399270..bc64bd2561382 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
@@ -1163,19 +1163,35 @@ define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
; GFX8-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_xor_v4i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v4i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_xor_v4i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v4i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v4i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v4i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v4i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1201,7 +1217,7 @@ define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1241,7 +1257,7 @@ define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -1250,42 +1266,97 @@ entry:
}
define i16 @test_vector_reduce_xor_v8i16(<8 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v8i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v8i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_xor_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v8i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_xor_v8i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_xor_v8i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX10-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v8i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v8i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v8i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v8i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v8i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1313,10 +1384,11 @@ define i16 @test_vector_reduce_xor_v8i16(<8 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_xor_v8i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1359,10 +1431,11 @@ define i16 @test_vector_reduce_xor_v8i16(<8 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX12-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1372,59 +1445,146 @@ entry:
}
define i16 @test_vector_reduce_xor_v16i16(<16 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v16i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v3, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v0, v1, v0
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_xor_b32_e32 v0, v3, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v16i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v1, v0
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v3, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_xor_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v16i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v8, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v8, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v8, v4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v8, v6
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v8, v7
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_xor_v16i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_xor_v16i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX10-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX10-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v16i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v16i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v16i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v16i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_alignbit_b32 v1, s4, v0, 16
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v16i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1458,14 +1618,17 @@ define i16 @test_vector_reduce_xor_v16i16(<16 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_xor_v16i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -1513,14 +1676,17 @@ define i16 @test_vector_reduce_xor_v16i16(<16 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX12-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -1623,29 +1789,53 @@ entry:
}
define i32 @test_vector_reduce_xor_v4i32(<4 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v4i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v4i32:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_xor_v4i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v4i32:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_xor_v4i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v4i32:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v4i32:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v4i32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v4i32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_xor_v4i32:
; GFX10: ; %bb.0: ; %entry
@@ -1679,802 +1869,1388 @@ entry:
}
define i32 @test_vector_reduce_xor_v8i32(<8 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v8i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_xor_v8i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_xor_v8i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_xor_v8i32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX10-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_vector_reduce_xor_v8i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX11-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: test_vector_reduce_xor_v8i32:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX12-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %v)
- ret i32 %res
-}
-
-define i32 @test_vector_reduce_xor_v16i32(<16 x i32> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v16i32:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_xor_v16i32:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_xor_v16i32:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_xor_v16i32:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_xor3_b32 v3, v3, v11, v7
-; GFX10-NEXT: v_xor3_b32 v2, v2, v10, v6
-; GFX10-NEXT: v_xor3_b32 v0, v0, v8, v4
-; GFX10-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX10-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_vector_reduce_xor_v16i32:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX11-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX11-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_xor3_b32 v3, v3, v11, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_xor3_b32 v2, v2, v10, v6
-; GFX11-NEXT: v_xor3_b32 v0, v0, v8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX11-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: test_vector_reduce_xor_v16i32:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX12-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX12-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_xor3_b32 v3, v3, v11, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_xor3_b32 v2, v2, v10, v6
-; GFX12-NEXT: v_xor3_b32 v0, v0, v8, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX12-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> %v)
- ret i32 %res
-}
-
-define i64 @test_vector_reduce_xor_v2i64(<2 x i64> %v) {
-; GFX7-SDAG-LABEL: test_vector_reduce_xor_v2i64:
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v8i32:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-GISEL-LABEL: test_vector_reduce_xor_v2i64:
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v8i32:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_xor_v2i64:
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v8i32:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GISEL-LABEL: test_vector_reduce_xor_v2i64:
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v8i32:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-SDAG-LABEL: test_vector_reduce_xor_v2i64:
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v8i32:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: test_vector_reduce_xor_v2i64:
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v8i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_xor_v2i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_vector_reduce_xor_v2i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: test_vector_reduce_xor_v2i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-entry:
- %res = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> %v)
- ret i64 %res
-}
-
-define i64 @test_vector_reduce_xor_v3i64(<3 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v3i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_xor_v3i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_xor_v3i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v8i32:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v8i32:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v4
-; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v3, v5
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v8i32:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX11-GISEL-LABEL: test_vector_reduce_xor_v8i32:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v4
-; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v3, v5
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v8i32:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX12-GISEL-LABEL: test_vector_reduce_xor_v8i32:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v4
-; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v3, v5
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
- %res = call i64 @llvm.vector.reduce.xor.v3i64(<3 x i64> %v)
- ret i64 %res
+ %res = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %v)
+ ret i32 %res
}
-define i64 @test_vector_reduce_xor_v4i64(<4 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v4i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+define i32 @test_vector_reduce_xor_v16i32(<16 x i32> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v16i32:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_xor_v4i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v16i32:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_xor_v4i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v16i32:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v16i32:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v16i32:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v16i32:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v16i32:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX10-SDAG-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX10-SDAG-NEXT: v_xor3_b32 v0, v0, v8, v4
+; GFX10-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v16i32:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v8, v4
+; GFX10-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v16i32:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX11-SDAG-NEXT: v_xor3_b32 v0, v0, v8, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_xor_v16i32:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v8, v4
+; GFX11-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v16i32:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX12-SDAG-NEXT: v_xor3_b32 v0, v0, v8, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_xor_v16i32:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v8, v4
+; GFX12-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> %v)
+ ret i32 %res
+}
+
+define i64 @test_vector_reduce_xor_v2i64(<2 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v2i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v2i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v2i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v2i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v2i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_xor_v4i64:
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v2i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: test_vector_reduce_xor_v2i64:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v7
; GFX10-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX10-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_xor_v4i64:
+; GFX11-LABEL: test_vector_reduce_xor_v2i64:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_xor_v4i64:
+; GFX12-LABEL: test_vector_reduce_xor_v2i64:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX12-NEXT: v_xor_b32_e32 v1, v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
- %res = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %v)
+ %res = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> %v)
ret i64 %res
}
-define i64 @test_vector_reduce_xor_v8i64(<8 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v8i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX7-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_vector_reduce_xor_v8i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_xor_v8i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+define i64 @test_vector_reduce_xor_v3i64(<3 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v4
+; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v3, v5
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v4
+; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v3, v5
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v3i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_xor_v3i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v4
+; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v3, v5
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i64 @llvm.vector.reduce.xor.v3i64(<3 x i64> %v)
+ ret i64 %res
+}
+
+define i64 @test_vector_reduce_xor_v4i64(<4 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v4i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v4i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v4i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v4i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v4i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v4i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v4i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v4i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v4i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_xor_v4i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v4i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_xor_v4i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %res = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %v)
+ ret i64 %res
+}
+
+define i64 @test_vector_reduce_xor_v8i64(<8 x i64> %v) {
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v8i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v8i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v8i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v8i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v8i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v8i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v8i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v8i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX10-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_xor_v8i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX10-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v8i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_xor_v8i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX11-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX11-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: test_vector_reduce_xor_v8i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX11-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_xor_v8i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX12-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX12-NEXT: v_xor_b32_e32 v7, v7, v15
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v8i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_xor_v8i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v7, v7, v15
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX12-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> %v)
ret i64 %res
}
define i64 @test_vector_reduce_xor_v16i64(<16 x i64> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v16i64:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v4, v4, v20
-; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX7-NEXT: v_xor_b32_e32 v12, v12, v28
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v16
-; GFX7-NEXT: v_xor_b32_e32 v8, v8, v24
-; GFX7-NEXT: v_xor_b32_e32 v6, v6, v22
-; GFX7-NEXT: v_xor_b32_e32 v14, v14, v30
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v18
-; GFX7-NEXT: v_xor_b32_e32 v10, v10, v26
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_xor_b32_e32 v5, v5, v21
-; GFX7-NEXT: v_xor_b32_e32 v13, v13, v29
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v17
-; GFX7-NEXT: v_xor_b32_e32 v9, v9, v25
-; GFX7-NEXT: v_xor_b32_e32 v7, v7, v23
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v19
-; GFX7-NEXT: v_xor_b32_e32 v11, v11, v27
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v2, v15, v20
-; GFX7-NEXT: v_xor_b32_e32 v2, v7, v2
-; GFX7-NEXT: v_xor_b32_e32 v2, v3, v2
-; GFX7-NEXT: v_xor_b32_e32 v1, v1, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v16i64:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX7-SDAG-NEXT: buffer_load_dword v20, off, s[0:3], s32
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v15, v20
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v7, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v2, v3, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v1, v1, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_xor_v16i64:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v4, v4, v20
-; GFX8-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX8-NEXT: v_xor_b32_e32 v12, v12, v28
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v16
-; GFX8-NEXT: v_xor_b32_e32 v8, v8, v24
-; GFX8-NEXT: v_xor_b32_e32 v6, v6, v22
-; GFX8-NEXT: v_xor_b32_e32 v14, v14, v30
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v18
-; GFX8-NEXT: v_xor_b32_e32 v10, v10, v26
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_xor_b32_e32 v5, v5, v21
-; GFX8-NEXT: v_xor_b32_e32 v13, v13, v29
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v17
-; GFX8-NEXT: v_xor_b32_e32 v9, v9, v25
-; GFX8-NEXT: v_xor_b32_e32 v7, v7, v23
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v19
-; GFX8-NEXT: v_xor_b32_e32 v11, v11, v27
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v2, v15, v20
-; GFX8-NEXT: v_xor_b32_e32 v2, v7, v2
-; GFX8-NEXT: v_xor_b32_e32 v2, v3, v2
-; GFX8-NEXT: v_xor_b32_e32 v1, v1, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v16i64:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX7-GISEL-NEXT: buffer_load_dword v16, off, s[0:3], s32
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v15, v16
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v7, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v2, v3, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_xor_v16i64:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: scratch_load_dword v31, off, s32
-; GFX9-NEXT: v_xor_b32_e32 v4, v4, v20
-; GFX9-NEXT: v_xor_b32_e32 v12, v12, v28
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v16
-; GFX9-NEXT: v_xor_b32_e32 v8, v8, v24
-; GFX9-NEXT: v_xor_b32_e32 v6, v6, v22
-; GFX9-NEXT: v_xor_b32_e32 v14, v14, v30
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v18
-; GFX9-NEXT: v_xor_b32_e32 v10, v10, v26
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_xor_b32_e32 v5, v5, v21
-; GFX9-NEXT: v_xor_b32_e32 v13, v13, v29
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v17
-; GFX9-NEXT: v_xor_b32_e32 v9, v9, v25
-; GFX9-NEXT: v_xor_b32_e32 v7, v7, v23
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v19
-; GFX9-NEXT: v_xor_b32_e32 v11, v11, v27
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_xor_b32_e32 v2, v15, v31
-; GFX9-NEXT: v_xor_b32_e32 v2, v7, v2
-; GFX9-NEXT: v_xor_b32_e32 v2, v3, v2
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v16i64:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX8-SDAG-NEXT: buffer_load_dword v20, off, s[0:3], s32
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v15, v20
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v7, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v2, v3, v2
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v1, v1, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_xor_v16i64:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v18
-; GFX10-NEXT: v_xor_b32_e32 v10, v10, v26
-; GFX10-NEXT: v_xor_b32_e32 v5, v5, v21
-; GFX10-NEXT: v_xor_b32_e32 v4, v4, v20
-; GFX10-NEXT: v_xor_b32_e32 v13, v13, v29
-; GFX10-NEXT: v_xor_b32_e32 v12, v12, v28
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v17
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v16
-; GFX10-NEXT: v_xor_b32_e32 v9, v9, v25
-; GFX10-NEXT: v_xor_b32_e32 v8, v8, v24
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v19
-; GFX10-NEXT: v_xor_b32_e32 v11, v11, v27
-; GFX10-NEXT: v_xor_b32_e32 v7, v7, v23
-; GFX10-NEXT: v_xor_b32_e32 v6, v6, v22
-; GFX10-NEXT: v_xor_b32_e32 v14, v14, v30
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX10-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v10, v15, v31
-; GFX10-NEXT: v_xor_b32_e32 v7, v7, v10
-; GFX10-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v16i64:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX8-GISEL-NEXT: buffer_load_dword v16, off, s[0:3], s32
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v15, v16
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v7, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, v3, v2
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, v1, v2
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_vector_reduce_xor_v16i64:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v18
-; GFX11-NEXT: v_xor_b32_e32 v10, v10, v26
-; GFX11-NEXT: v_xor_b32_e32 v5, v5, v21
-; GFX11-NEXT: v_xor_b32_e32 v4, v4, v20
-; GFX11-NEXT: v_xor_b32_e32 v13, v13, v29
-; GFX11-NEXT: v_xor_b32_e32 v12, v12, v28
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v17
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v16
-; GFX11-NEXT: v_xor_b32_e32 v9, v9, v25
-; GFX11-NEXT: v_xor_b32_e32 v8, v8, v24
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v19
-; GFX11-NEXT: v_xor_b32_e32 v11, v11, v27
-; GFX11-NEXT: v_xor_b32_e32 v7, v7, v23
-; GFX11-NEXT: v_xor_b32_e32 v6, v6, v22
-; GFX11-NEXT: v_xor_b32_e32 v14, v14, v30
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX11-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX11-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX11-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v10, v15, v31
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_xor_b32_e32 v7, v7, v10
-; GFX11-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_xor_v16i64:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v15, v31
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v7, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v2, v3, v2
+; GFX9-SDAG-NEXT: v_xor_b32_e32 v1, v1, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_vector_reduce_xor_v16i64:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v18
-; GFX12-NEXT: v_xor_b32_e32 v10, v10, v26
-; GFX12-NEXT: v_xor_b32_e32 v5, v5, v21
-; GFX12-NEXT: v_xor_b32_e32 v4, v4, v20
-; GFX12-NEXT: v_xor_b32_e32 v13, v13, v29
-; GFX12-NEXT: v_xor_b32_e32 v12, v12, v28
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v17
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v16
-; GFX12-NEXT: v_xor_b32_e32 v9, v9, v25
-; GFX12-NEXT: v_xor_b32_e32 v8, v8, v24
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v19
-; GFX12-NEXT: v_xor_b32_e32 v11, v11, v27
-; GFX12-NEXT: v_xor_b32_e32 v7, v7, v23
-; GFX12-NEXT: v_xor_b32_e32 v6, v6, v22
-; GFX12-NEXT: v_xor_b32_e32 v14, v14, v30
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v10
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v11
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v9
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v8
-; GFX12-NEXT: v_xor_b32_e32 v5, v5, v13
-; GFX12-NEXT: v_xor_b32_e32 v4, v4, v12
-; GFX12-NEXT: v_xor_b32_e32 v6, v6, v14
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v10, v15, v31
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_xor_b32_e32 v7, v7, v10
-; GFX12-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_xor_v16i64:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: scratch_load_dword v31, off, s32
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v15, v31
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v7, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, v3, v2
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v1, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_xor_v16i64:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v10, v15, v31
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v7, v7, v10
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_xor_v16i64:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX10-GISEL-NEXT: v_xor3_b32 v6, v6, v22, v14
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v16, v8
+; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v17, v9
+; GFX10-GISEL-NEXT: v_xor3_b32 v4, v4, v20, v12
+; GFX10-GISEL-NEXT: v_xor3_b32 v5, v5, v21, v13
+; GFX10-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX10-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v15, v15, v31
+; GFX10-GISEL-NEXT: v_xor3_b32 v7, v7, v23, v15
+; GFX10-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v16i64:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v10, v15, v31
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v7, v7, v10
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_vector_reduce_xor_v16i64:
+; GFX11-GISEL: ; %bb.0: ; %entry
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX11-GISEL-NEXT: v_xor3_b32 v6, v6, v22, v14
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v16, v8
+; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v17, v9
+; GFX11-GISEL-NEXT: v_xor3_b32 v4, v4, v20, v12
+; GFX11-GISEL-NEXT: v_xor3_b32 v5, v5, v21, v13
+; GFX11-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_xor_b32_e32 v15, v15, v31
+; GFX11-GISEL-NEXT: v_xor3_b32 v7, v7, v23, v15
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v16i64:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v5, v5, v21
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v4, v4, v20
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v17
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v16
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v7, v7, v23
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v6, v6, v22
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v10
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v11
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v9
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v8
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v5, v5, v13
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v4, v4, v12
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v6, v6, v14
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v10, v15, v31
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v7, v7, v10
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: test_vector_reduce_xor_v16i64:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v14, v14, v30
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v3, v3, v19
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v8, v8, v24
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v9, v9, v25
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v10, v10, v26
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v11, v11, v27
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v12, v12, v28
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v13, v13, v29
+; GFX12-GISEL-NEXT: v_xor3_b32 v6, v6, v22, v14
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v16, v8
+; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v17, v9
+; GFX12-GISEL-NEXT: v_xor3_b32 v4, v4, v20, v12
+; GFX12-GISEL-NEXT: v_xor3_b32 v5, v5, v21, v13
+; GFX12-GISEL-NEXT: v_xor3_b32 v2, v2, v10, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_xor_b32_e32 v15, v15, v31
+; GFX12-GISEL-NEXT: v_xor3_b32 v7, v7, v23, v15
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_xor3_b32 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> %v)
ret i64 %res
More information about the llvm-branch-commits
mailing list