[llvm] AMDGPU/GlobalISel: Handle G_BITCAST for 16 bit extendedLLTs (PR #208750)
Petar Avramovic via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 02:38:26 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/208750
>From f3b7bf5f46b7ae9eb6a025a3410b3d71974b6f9a Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Thu, 16 Jul 2026 17:47:29 +0200
Subject: [PATCH] AMDGPU/GlobalISel: Handle G_BITCAST for 16 bit extendedLLTs
Handle bitcast between i16 and f16/bf16.
For true16 this was already legal, make it legal in regbanklegalize as well.
For non-true16 widen it using G_ANYEXT to i32 and G_TRUNC to dst.
The "i32 G_ANYEXT f16/bf16" and "f16/bf16 G_TRUNC i32" are already legal,
for example these are generated by common CallLowering argument lowering.
---
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 20 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 8 +
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 2 +
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 2 +
.../AMDGPU/GlobalISel/atomic_load_flat.ll | 60 +-
.../AMDGPU/GlobalISel/atomic_load_global.ll | 22 +-
.../AMDGPU/GlobalISel/atomic_load_local_2.ll | 8 +-
.../GlobalISel/clamp-fmed3-const-combine.ll | 12 +-
.../GlobalISel/clamp-minmax-const-combine.ll | 24 +-
.../combine-binop-s64-with-s32-mask.mir | 60 +-
.../GlobalISel/combine-fma-add-ext-fma.ll | 18 +-
.../GlobalISel/combine-fma-add-fma-mul.ll | 12 +-
.../AMDGPU/GlobalISel/combine-fma-add-mul.ll | 12 +-
.../GlobalISel/combine-fma-sub-ext-mul.ll | 4 +-
.../GlobalISel/combine-fma-sub-ext-neg-mul.ll | 12 +-
.../AMDGPU/GlobalISel/combine-fma-sub-mul.ll | 16 +-
.../GlobalISel/combine-fma-sub-neg-mul.ll | 16 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll | 53 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll | 8 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fcmp.ll | 234 +-
.../CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll | 532 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll | 6 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmad.ll | 18 +-
.../fmamix-constant-bus-violation.ll | 29 +-
.../GlobalISel/fmed3-min-max-const-combine.ll | 14 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll | 10 +-
.../AMDGPU/GlobalISel/fmin3-fmax3-combine.ll | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll | 8 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll | 53 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll | 36 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll | 3049 ++++---
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 5383 ++++++------
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 5260 ++++++------
llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll | 8 +-
.../AMDGPU/GlobalISel/intrinsic-trunc.ll | 4 +-
.../AMDGPU/GlobalISel/legalize-fptrunc.mir | 119 +-
.../legalize-llvm.amdgcn.image.load.2d.d16.ll | 1041 ++-
.../legalize-llvm.amdgcn.image.sample.a16.ll | 1652 ++--
...galize-llvm.amdgcn.image.sample.g16.a16.ll | 80 +-
.../legalize-llvm.amdgcn.image.sample.g16.ll | 412 +-
...legalize-llvm.amdgcn.image.store.2d.d16.ll | 100 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.fract.ll | 9 +-
.../GlobalISel/llvm.amdgcn.frexp.mant.ll | 2 +-
.../llvm.amdgcn.image.gather4.a16.dim.ll | 1441 ++--
.../llvm.amdgcn.image.sample.cd.g16.ll | 48 +-
.../llvm.amdgcn.image.sample.g16.ll | 236 +-
.../llvm.amdgcn.image.store.2d.d16.ll | 36 +-
.../GlobalISel/llvm.amdgcn.intersect_ray.ll | 302 +-
.../llvm.amdgcn.raw.buffer.load.tfe.ll | 241 +-
...llvm.amdgcn.raw.buffer.store.format.f16.ll | 86 +-
.../llvm.amdgcn.raw.buffer.store.ll | 24 +-
.../llvm.amdgcn.raw.ptr.buffer.load.ll | 4 +-
....amdgcn.raw.ptr.buffer.store.format.f16.ll | 92 +-
.../llvm.amdgcn.raw.ptr.buffer.store.ll | 24 +-
.../llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll | 656 +-
.../llvm.amdgcn.raw.tbuffer.store.f16.ll | 598 +-
...vm.amdgcn.struct.buffer.load.format.f16.ll | 48 +-
.../llvm.amdgcn.struct.buffer.load.tfe.ll | 249 +-
...m.amdgcn.struct.buffer.store.format.f16.ll | 280 +-
....struct.ptr.buffer.atomic.fadd-with-ret.ll | 2 +-
...mdgcn.struct.ptr.buffer.load.format.f16.ll | 46 +-
...dgcn.struct.ptr.buffer.store.format.f16.ll | 228 +-
.../CodeGen/AMDGPU/GlobalISel/llvm.powi.ll | 39 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll | 2 +
.../madmix-constant-bus-violation.ll | 29 +-
.../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 59 +-
.../AMDGPU/GlobalISel/select-to-fmin-fmax.ll | 4 +-
.../AMDGPU/GlobalISel/shl-ext-reduce.ll | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll | 2 +
llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll | 4 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 57 +-
.../AMDGPU/GlobalISel/store-local.96.ll | 2 +-
.../AMDGPU/GlobalISel/strict_fma.f16.ll | 48 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll | 8 +-
.../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 18 +-
.../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 18 +-
llvm/test/CodeGen/AMDGPU/add-max.ll | 2 +-
.../CodeGen/AMDGPU/amdgcn-sin-cos-f16-f32.ll | 4 +-
.../test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll | 482 +-
llvm/test/CodeGen/AMDGPU/bfi_int.ll | 4 +-
llvm/test/CodeGen/AMDGPU/bitop3.ll | 110 +-
llvm/test/CodeGen/AMDGPU/bitreverse.ll | 4 +-
.../CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll | 2 +-
.../build-vector-packed-partial-undef.ll | 390 +-
llvm/test/CodeGen/AMDGPU/call-args-inreg.ll | 4 +-
llvm/test/CodeGen/AMDGPU/call-return-types.ll | 4 +-
.../AMDGPU/commute-compares-scalar-float.ll | 42 +-
llvm/test/CodeGen/AMDGPU/ctlz.ll | 2 +-
llvm/test/CodeGen/AMDGPU/ctpop64.ll | 2 +-
llvm/test/CodeGen/AMDGPU/cttz.ll | 2 +-
.../CodeGen/AMDGPU/dagcombine-fmul-sel.ll | 230 +-
.../AMDGPU/diverge-interp-mov-lower.ll | 6 +-
.../test/CodeGen/AMDGPU/dynamic_stackalloc.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fadd.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll | 13 +-
llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll | 144 +-
llvm/test/CodeGen/AMDGPU/fma.f16.ll | 145 +-
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 35 +-
llvm/test/CodeGen/AMDGPU/fmaxnum.ll | 131 +-
.../test/CodeGen/AMDGPU/fmed3-cast-combine.ll | 179 +-
llvm/test/CodeGen/AMDGPU/fmed3.ll | 12 +-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 35 +-
llvm/test/CodeGen/AMDGPU/fminnum.ll | 131 +-
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll | 520 +-
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll | 14 +-
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 171 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll | 73 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll | 20 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 560 +-
.../AMDGPU/fsub-as-fneg-src-modifier.ll | 226 +-
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 10 +-
.../CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll | 84 +-
.../isel-amdgpu-cs-chain-preserve-cc.ll | 41 +-
llvm/test/CodeGen/AMDGPU/literal64.ll | 4 +-
.../test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll | 189 +-
.../CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll | 308 +-
.../llvm.amdgcn.cluster.load.async.to.lds.ll | 2 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll | 32 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll | 154 +-
.../AMDGPU/llvm.amdgcn.div.fixup.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll | 188 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll | 287 +-
.../AMDGPU/llvm.amdgcn.flat.prefetch.ll | 2 +-
.../AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll | 2 +-
.../AMDGPU/llvm.amdgcn.frexp.exp.f16.ll | 2 +-
.../llvm.amdgcn.global.load.async.to.lds.ll | 2 +-
.../AMDGPU/llvm.amdgcn.global.prefetch.ll | 2 +-
...llvm.amdgcn.global.store.async.from.lds.ll | 2 +-
.../llvm.amdgcn.image.sample.g16.a16.dim.ll | 291 +-
.../AMDGPU/llvm.amdgcn.image.sample.noret.ll | 48 +-
.../CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll | 18 +-
.../AMDGPU/llvm.amdgcn.interp.inreg.ll | 4 +-
.../AMDGPU/llvm.amdgcn.intersect_ray.ll | 283 +-
.../CodeGen/AMDGPU/llvm.amdgcn.is.private.ll | 2 +-
.../llvm.amdgcn.load.monitor.gfx1250.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.ll | 2 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.permlane.ll | 7431 ++++++++---------
.../llvm.amdgcn.raw.tbuffer.store.d16.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll | 249 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll | 213 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll | 213 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll | 251 +-
.../CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll | 4 +-
.../llvm.amdgcn.struct.tbuffer.store.d16.ll | 4 +-
.../AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.writelane.ll | 20 +-
llvm/test/CodeGen/AMDGPU/llvm.exp.ll | 534 +-
llvm/test/CodeGen/AMDGPU/llvm.exp10.ll | 543 +-
llvm/test/CodeGen/AMDGPU/llvm.exp2.ll | 31 +-
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 45 +-
llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll | 12 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.f16.ll | 1797 ++--
llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll | 108 +-
llvm/test/CodeGen/AMDGPU/llvm.log.ll | 227 +-
llvm/test/CodeGen/AMDGPU/llvm.log10.ll | 227 +-
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 33 +-
llvm/test/CodeGen/AMDGPU/llvm.modf.ll | 70 +-
llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll | 12 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/lround.ll | 17 +-
llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll | 323 +-
llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll | 23 +-
llvm/test/CodeGen/AMDGPU/mad-mix.ll | 904 +-
llvm/test/CodeGen/AMDGPU/memset-pattern.ll | 922 +-
llvm/test/CodeGen/AMDGPU/minimummaximum.ll | 91 +-
llvm/test/CodeGen/AMDGPU/minmax.ll | 113 +-
.../AMDGPU/pseudo-scalar-transcendental.ll | 103 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll | 22 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll | 4 +-
llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll | 4 +-
.../CodeGen/AMDGPU/scale-offset-global.ll | 8 +-
llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll | 2 +-
llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll | 10 +-
llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll | 10 +-
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | 10 +-
llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll | 122 +-
llvm/test/CodeGen/AMDGPU/v_pack.ll | 6 +-
llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll | 4 +-
185 files changed, 24086 insertions(+), 19707 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 19e15b1ae0efb..2f607327ef152 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3640,6 +3640,26 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
return Legalized;
}
+ case TargetOpcode::G_BITCAST:
+ if (WideTy.isVector())
+ return UnableToLegalize;
+ Observer.changingInstr(MI);
+ if (TypeIdx == 0)
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
+ else
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
+ Observer.changedInstr(MI);
+
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ if (MRI.getType(Dst) == MRI.getType(Src)) {
+ Observer.changingAllUsesOfReg(MRI, Dst);
+ MRI.replaceRegWith(Dst, Src);
+ Observer.finishedChangingAllUsesOfReg();
+ MI.eraseFromParent();
+ }
+
+ return Legalized;
}
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index a51997f54cd93..aeaff1acea1b1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -293,6 +293,10 @@ static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
};
}
+const LLT I16 = LLT::integer(16);
+constexpr LLT F16 = LLT::float16();
+constexpr LLT BF16 = LLT::bfloat16();
+
constexpr LLT S1 = LLT::scalar(1);
constexpr LLT S8 = LLT::scalar(8);
constexpr LLT S16 = LLT::scalar(16);
@@ -926,6 +930,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
getActionDefinitionsBuilder(G_BITCAST)
// Don't worry about the size constraint.
.legalIf(all(isRegisterClassType(ST, 0), isRegisterClassType(ST, 1)))
+ .widenScalarIf(all(typeInSet(0, {I16, F16, BF16}), isScalar(1)),
+ changeTo(0, LLT::integer(32)))
+ .widenScalarIf(all(isScalar(0), typeInSet(1, {I16, F16, BF16})),
+ changeTo(1, LLT::integer(32)))
.lower();
getActionDefinitionsBuilder(G_CONSTANT)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 7f7225ad6d311..df5b5719afd05 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -753,6 +753,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{DivBRC}, {{VgprBRC}, {VgprBRC}}});
addRulesForGOpcs({G_BITCAST})
+ .Any({{UniS16}, {{Sgpr16}, {Sgpr16}}})
+ .Any({{DivS16}, {{Vgpr16}, {Vgpr16}}})
.Any({{UniBRC}, {{SgprBRC}, {SgprBRC}}})
.Any({{DivBRC}, {{VgprBRC}, {VgprBRC}}});
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index a7c6d8b158c70..eb91938582fb6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -731,6 +731,7 @@ define amdgpu_ps i16 @s_ashr_i16_15(i16 inreg %value) {
define amdgpu_ps half @ashr_i16_sv(i16 inreg %value, i16 %amount) {
; GFX6-LABEL: ashr_i16_sv:
; GFX6: ; %bb.0:
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: s_sext_i32_i16 s0, s0
; GFX6-NEXT: v_ashr_i32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
@@ -767,6 +768,7 @@ define amdgpu_ps half @ashr_i16_sv(i16 inreg %value, i16 %amount) {
define amdgpu_ps half @ashr_i16_vs(i16 %value, i16 inreg %amount) {
; GFX6-LABEL: ashr_i16_vs:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX6-NEXT: v_ashrrev_i32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_flat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_flat.ll
index df8e59c8c03f6..712785a1b237b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_flat.ll
@@ -121,28 +121,12 @@ define bfloat @atomic_load_flat_monotonic_bf16(ptr %ptr) {
}
define i32 @atomic_load_flat_monotonic_f16_zext_to_i32(ptr %ptr) {
-; GFX7-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
-; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: flat_load_ushort v0, v[0:1] glc
-; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GCN-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: flat_load_ushort v0, v[0:1] glc
+; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64 s[30:31]
%load = load atomic half, ptr %ptr monotonic, align 2
%cast = bitcast half %load to i16
%ext = zext i16 %cast to i32
@@ -150,28 +134,12 @@ define i32 @atomic_load_flat_monotonic_f16_zext_to_i32(ptr %ptr) {
}
define i32 @atomic_load_flat_monotonic_bf16_zext_to_i32(ptr %ptr) {
-; GFX7-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
-; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: flat_load_ushort v0, v[0:1] glc
-; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GCN-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: flat_load_ushort v0, v[0:1] glc
+; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64 s[30:31]
%load = load atomic bfloat, ptr %ptr monotonic, align 2
%cast = bitcast bfloat %load to i16
%ext = zext i16 %cast to i32
@@ -270,7 +238,7 @@ define <2 x i16> @atomic_load_flat_monotonic_i16_d16_lo_vector_insert(ptr %ptr,
; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_flat_monotonic_i16_d16_lo_vector_insert:
@@ -279,7 +247,7 @@ define <2 x i16> @atomic_load_flat_monotonic_i16_d16_lo_vector_insert(ptr %ptr,
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_flat_monotonic_i16_d16_lo_vector_insert:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll
index 5ce7c8929c9ac..e4d06bcc2cb1b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll
@@ -367,8 +367,7 @@ define i32 @atomic_load_global_monotonic_f16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -385,7 +384,6 @@ define i32 @atomic_load_global_monotonic_f16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_global_monotonic_f16_zext_to_i32:
@@ -393,7 +391,6 @@ define i32 @atomic_load_global_monotonic_f16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v0, v[0:1], off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic half, ptr addrspace(1) %ptr monotonic, align 2
%cast = bitcast half %load to i16
@@ -407,8 +404,7 @@ define i32 @atomic_load_global_monotonic_bf16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -425,7 +421,6 @@ define i32 @atomic_load_global_monotonic_bf16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_global_monotonic_bf16_zext_to_i32:
@@ -433,7 +428,6 @@ define i32 @atomic_load_global_monotonic_bf16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v0, v[0:1], off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic bfloat, ptr addrspace(1) %ptr monotonic, align 2
%cast = bitcast bfloat %load to i16
@@ -488,8 +482,7 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_hi_vector_insert(ptr addr
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -586,12 +579,11 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_lo_vector_insert(ptr addr
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_global_monotonic_i16_d16_lo_vector_insert:
@@ -600,7 +592,7 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_lo_vector_insert(ptr addr
; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_global_monotonic_i16_d16_lo_vector_insert:
@@ -609,7 +601,7 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_lo_vector_insert(ptr addr
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_global_monotonic_i16_d16_lo_vector_insert:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_local_2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_local_2.ll
index 2f6ae98abaf91..5e41d2388f828 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_local_2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_local_2.ll
@@ -297,7 +297,6 @@ define i32 @atomic_load_local_monotonic_f16_zext_to_i32(ptr addrspace(3) %ptr) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_u16 v0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_local_monotonic_f16_zext_to_i32:
@@ -305,7 +304,6 @@ define i32 @atomic_load_local_monotonic_f16_zext_to_i32(ptr addrspace(3) %ptr) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u16 v0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic half, ptr addrspace(3) %ptr monotonic, align 2
%cast = bitcast half %load to i16
@@ -328,7 +326,6 @@ define i32 @atomic_load_local_monotonic_bf16_zext_to_i32(ptr addrspace(3) %ptr)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_u16 v0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_local_monotonic_bf16_zext_to_i32:
@@ -336,7 +333,6 @@ define i32 @atomic_load_local_monotonic_bf16_zext_to_i32(ptr addrspace(3) %ptr)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u16 v0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic bfloat, ptr addrspace(3) %ptr monotonic, align 2
%cast = bitcast bfloat %load to i16
@@ -458,7 +454,7 @@ define <2 x i16> @atomic_load_local_monotonic_i16_d16_lo_vector_insert(ptr addrs
; GFX7-NEXT: ds_read_u16 v0, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_local_monotonic_i16_d16_lo_vector_insert:
@@ -468,7 +464,7 @@ define <2 x i16> @atomic_load_local_monotonic_i16_d16_lo_vector_insert(ptr addrs
; GFX8-NEXT: ds_read_u16 v0, v0
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_local_monotonic_i16_d16_lo_vector_insert:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll
index a7b1aa105c8de..a33785e16722a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define float @test_fmed3_f32_known_nnan_ieee_true(float %a) #0 {
; GFX10-LABEL: test_fmed3_f32_known_nnan_ieee_true:
@@ -35,13 +35,13 @@ define half @test_fmed3_f16_known_nnan_ieee_false(half %a) #1 {
; GFX10-LABEL: test_fmed3_f16_known_nnan_ieee_false:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_f16_e64 v0, v0, v0 clamp
+; GFX10-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_fmed3_f16_known_nnan_ieee_false:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_add_f16_e64 v0, v0, v0 clamp
+; GFX1170-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_fmed3_f16_known_nnan_ieee_false:
@@ -61,7 +61,7 @@ define half @test_fmed3_f16_known_nnan_ieee_false(half %a) #1 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_f16_e64 v0, v0, v0 clamp
+; GFX12-FAKE16-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%fmul = fmul half %a, 2.0
%fmed = call nnan half @llvm.amdgcn.fmed3.f16(half %fmul, half 0.0, half 1.0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll
index 7378615fae19c..3ffaaa6376baa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define float @test_min_max_ValK0_K1_f32(float %a) #0 {
; GFX10-LABEL: test_min_max_ValK0_K1_f32:
@@ -65,13 +65,13 @@ define half @test_min_K1max_ValK0_f16(half %a) #2 {
; GFX10-LABEL: test_min_K1max_ValK0_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_f16_e64 v0, v0, v0 clamp
+; GFX10-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_min_K1max_ValK0_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_add_f16_e64 v0, v0, v0 clamp
+; GFX1170-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_min_K1max_ValK0_f16:
@@ -91,7 +91,7 @@ define half @test_min_K1max_ValK0_f16(half %a) #2 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_f16_e64 v0, v0, v0 clamp
+; GFX12-FAKE16-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%fmul = fmul half %a, 2.0
%maxnum = call half @llvm.maxnum.f16(half %fmul, half 0.0)
@@ -217,17 +217,13 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX10-LABEL: test_max_K0min_ValK1_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_f16_e32 v0, v0, v0
-; GFX10-NEXT: v_min_f16_e32 v0, 1.0, v0
-; GFX10-NEXT: v_max_f16_e32 v0, 0, v0
+; GFX10-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_max_K0min_ValK1_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_add_f16_e32 v0, v0, v0
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-NEXT: v_minmax_num_f16 v0, v0, 1.0, 0
+; GFX1170-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_max_K0min_ValK1_f16:
@@ -247,9 +243,7 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_f16_e32 v0, v0, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_minmax_num_f16 v0, v0, 1.0, 0
+; GFX12-FAKE16-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%fmul = fmul half %a, 2.0
%minnum = call nnan half @llvm.minnum.f16(half %fmul, half 1.0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
index e680331330ce8..1c42ae83694d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
@@ -12,9 +12,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -33,9 +33,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -54,10 +54,10 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -65536
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND]](s32), [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -65536
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -96,8 +96,8 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[ZEXT]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -116,8 +116,8 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[ZEXT]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -136,10 +136,10 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UV]](s32), [[AND]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UV]](i32), [[AND]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -202,9 +202,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](i32), [[C]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -223,9 +223,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](s32), [[C]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](i32), [[C]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -244,9 +244,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@@ -265,9 +265,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll
index e63afa4a1ff2a..5909e89b9199e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fadd (fma x, y, (fpext (fmul u, v))), z) -> (fma x, y, (fma (fpext u), (fpext v), z))
define amdgpu_vs float @test_f16_f32_add_fma_ext_mul(float %x, float %y, float %z, half %u, half %v) {
@@ -45,9 +45,8 @@ define amdgpu_vs float @test_f16_f32_add_fma_ext_mul(float %x, float %y, float %
define amdgpu_vs float @test_f16_f32_add_ext_fma_mul(half %x, half %y, float %z, half %u, half %v) {
; GFX9-DENORM-LABEL: test_f16_f32_add_ext_fma_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
-; GFX9-DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
-; GFX9-DENORM-NEXT: v_mac_f16_e32 v3, v0, v1
-; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v3, 1.0, v2 op_sel_hi:[1,1,0]
+; GFX9-DENORM-NEXT: v_mad_mix_f32 v2, v3, v4, v2 op_sel_hi:[1,1,0]
+; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: test_f16_f32_add_ext_fma_mul:
@@ -119,9 +118,8 @@ define amdgpu_vs float @test_f16_f32_add_fma_ext_mul_rhs(float %x, float %y, flo
define amdgpu_vs float @test_f16_f32_add_ext_fma_mul_rhs(float %x, half %y, half %z, half %u, half %v) {
; GFX9-DENORM-LABEL: test_f16_f32_add_ext_fma_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
-; GFX9-DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
-; GFX9-DENORM-NEXT: v_mac_f16_e32 v3, v1, v2
-; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v3, 1.0, v0 op_sel_hi:[1,1,0]
+; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v3, v4, v0 op_sel_hi:[1,1,0]
+; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: test_f16_f32_add_ext_fma_mul_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
index 97bcd73228eba..ce4de6e628573 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
; fadd (fma a, b, (fmul c, d)), e --> fma a, b, (fma c, d, e)
; fadd e, (fma a, b, (fmul c, d)) --> fma a, b, (fma c, d, e)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-mul.ll
index 7dd9791a42fe2..7a8886b5c0cf7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-mul.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
define float @test_f32_add_mul(float %x, float %y, float %z) {
; GFX9-LABEL: test_f32_add_mul:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-mul.ll
index be091bb30da68..55cf23b88306e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-mul.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z))
define amdgpu_vs float @test_f16_to_f32_sub_ext_mul(half %x, half %y, float %z) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-neg-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-neg-mul.ll
index 85ec8c3e2facc..4f06c0f436a00 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-neg-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-neg-mul.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fsub (fpext (fneg (fmul, x, y))), z) -> (fneg (fma (fpext x), (fpext y), z))
define amdgpu_vs float @test_f16_to_f32_sub_ext_neg_mul(half %x, half %y, float %z) {
@@ -45,12 +45,12 @@ entry:
define amdgpu_vs float @test_f16_to_f32_sub_ext_neg_mul2(float %x, half %y, half %z) {
; GFX9-DENORM-LABEL: test_f16_to_f32_sub_ext_neg_mul2:
; GFX9-DENORM: ; %bb.0: ; %entry
-; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
+; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-DENORM-LABEL: test_f16_to_f32_sub_ext_neg_mul2:
; GFX10-DENORM: ; %bb.0: ; %entry
-; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
+; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX10-DENORM-NEXT: ; return to shader part epilog
entry:
%a = fmul fast half %y, %z
@@ -64,12 +64,12 @@ entry:
define amdgpu_vs float @test_f16_to_f32_sub_neg_ext_mul2(float %x, half %y, half %z) {
; GFX9-DENORM-LABEL: test_f16_to_f32_sub_neg_ext_mul2:
; GFX9-DENORM: ; %bb.0: ; %entry
-; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
+; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-DENORM-LABEL: test_f16_to_f32_sub_neg_ext_mul2:
; GFX10-DENORM: ; %bb.0: ; %entry
-; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
+; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX10-DENORM-NEXT: ; return to shader part epilog
entry:
%a = fmul fast half %y, %z
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll
index 4ac35f578eebb..113ba59751364 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll
@@ -1,14 +1,14 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
; fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
; fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-neg-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-neg-mul.ll
index 0b28f8323cf9b..cb97224578b82 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-neg-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-neg-mul.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
define float @test_f32_sub_ext_neg_mul(float %x, float %y, float %z) {
@@ -63,7 +63,7 @@ define half @test_f16_sub_ext_neg_mul(half %x, half %y, half %z) {
; GFX9-CONTRACT-LABEL: test_f16_sub_ext_neg_mul:
; GFX9-CONTRACT: ; %bb.0: ; %entry
; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-CONTRACT-NEXT: v_fma_f16 v0, -v0, v1, -v2
+; GFX9-CONTRACT-NEXT: v_fma_f16 v0, v0, -v1, -v2
; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f16_sub_ext_neg_mul:
@@ -82,7 +82,7 @@ define half @test_f16_sub_ext_neg_mul(half %x, half %y, half %z) {
; GFX10-CONTRACT-LABEL: test_f16_sub_ext_neg_mul:
; GFX10-CONTRACT: ; %bb.0: ; %entry
; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-CONTRACT-NEXT: v_fma_f16 v0, -v0, v1, -v2
+; GFX10-CONTRACT-NEXT: v_fma_f16 v0, v0, -v1, -v2
; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f16_sub_ext_neg_mul:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
index 1cd43aecf508f..fb81eba869caa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
define amdgpu_ps void @v_fabs_f16(half %in, ptr addrspace(1) %out) {
; GCN-LABEL: v_fabs_f16:
@@ -13,27 +13,44 @@ define amdgpu_ps void @v_fabs_f16(half %in, ptr addrspace(1) %out) {
ret void
}
define amdgpu_ps void @s_fabs_f16(half inreg %in, ptr addrspace(1) %out) {
-; GCN-LABEL: s_fabs_f16:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_and_b32 s0, s0, 0x7fff
-; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: global_store_b16 v[0:1], v2, off
-; GCN-NEXT: s_endpgm
+; GFX11-LABEL: s_fabs_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_and_b32_e64 v2, 0x7fff, s0
+; GFX11-NEXT: global_store_b16 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: s_fabs_f16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%fabs = call half @llvm.fabs.f16(half %in)
store half %fabs, ptr addrspace(1) %out
ret void
}
define amdgpu_ps void @s_fabs_f16_salu_use(half inreg %in, i32 inreg %val, ptr addrspace(1) %out) {
-; GCN-LABEL: s_fabs_f16_salu_use:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_and_b32 s0, s0, 0x7fff
-; GCN-NEXT: s_cmp_eq_u32 s1, 0
-; GCN-NEXT: s_cselect_b32 s1, -1, 0
-; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, s0, s1
-; GCN-NEXT: global_store_b16 v[0:1], v2, off
-; GCN-NEXT: s_endpgm
+; GFX11-LABEL: s_fabs_f16_salu_use:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_and_b32_e64 v2, 0x7fff, s0
+; GFX11-NEXT: s_cmp_eq_u32 s1, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: s_cselect_b32 s0, s0, 0
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-NEXT: global_store_b16 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: s_fabs_f16_salu_use:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX12-NEXT: s_cmp_eq_u32 s1, 0
+; GFX12-NEXT: s_cselect_b32 s0, s0, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%fabs = call half @llvm.fabs.f16(half %in)
%cond = icmp eq i32 %val, 0
%sel = select i1 %cond, half %fabs, half 0.0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
index 61f6b595e62fd..808bb42378841 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
define amdgpu_ps half @fadd_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-FAKE16-LABEL: fadd_s16_uniform:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
index 1496aaff4d2f7..a25a30fa5a8a3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=FIJI %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 < %s | FileCheck -check-prefix=FIJI %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GFX12 %s
define float @fceil_s(float inreg %val) {
; FIJI-LABEL: fceil_s:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcmp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcmp.ll
index 839f8a398c3d0..6011841643f15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcmp.ll
@@ -1,53 +1,94 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu10.10 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GFX10
-; RUN: llc -mtriple=amdgpu12.00 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GFX12
+; RUN: llc -mtriple=amdgpu10.10 -global-isel < %s | FileCheck %s --check-prefix=GFX10
+; RUN: llc -mtriple=amdgpu12.00 -global-isel < %s | FileCheck %s --check-prefix=GFX12
define void @fcmp_f16_uniform(half inreg %a, half inreg %b, ptr %p) {
; GFX10-LABEL: fcmp_f16_uniform:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_cmp_eq_f16_e64 s4, s16, s17
; GFX10-NEXT: v_cmp_gt_f16_e64 s5, s16, s17
; GFX10-NEXT: v_cmp_ge_f16_e64 s6, s16, s17
+; GFX10-NEXT: v_cmp_lt_f16_e64 s7, s16, s17
; GFX10-NEXT: v_cmp_le_f16_e64 s8, s16, s17
-; GFX10-NEXT: v_cmp_o_f16_e64 s9, s16, s17
-; GFX10-NEXT: v_cmp_nle_f16_e64 s10, s16, s17
-; GFX10-NEXT: v_cmp_nge_f16_e64 s11, s16, s17
-; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX10-NEXT: v_cmp_neq_f16_e64 s12, s16, s17
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: v_cmp_lg_f16_e64 s9, s16, s17
+; GFX10-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: v_cmp_o_f16_e64 s10, s16, s17
; GFX10-NEXT: s_cselect_b32 s5, 1, 0
-; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX10-NEXT: v_cmp_eq_f16_e64 s4, s16, s17
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
+; GFX10-NEXT: v_cmp_nlg_f16_e64 s11, s16, s17
; GFX10-NEXT: s_cselect_b32 s6, 1, 0
-; GFX10-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s7, 0
+; GFX10-NEXT: v_cmp_nle_f16_e64 s12, s16, s17
+; GFX10-NEXT: s_cselect_b32 s7, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: v_cmp_nlt_f16_e64 s13, s16, s17
; GFX10-NEXT: s_cselect_b32 s8, 1, 0
-; GFX10-NEXT: s_and_b32 s9, s9, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: v_cmp_nge_f16_e64 s14, s16, s17
; GFX10-NEXT: s_cselect_b32 s9, 1, 0
-; GFX10-NEXT: s_and_b32 s10, s10, exec_lo
-; GFX10-NEXT: v_cmp_lt_f16_e64 s7, s16, s17
+; GFX10-NEXT: s_cmp_lg_u32 s10, 0
+; GFX10-NEXT: v_cmp_ngt_f16_e64 s15, s16, s17
; GFX10-NEXT: s_cselect_b32 s10, 1, 0
-; GFX10-NEXT: s_and_b32 s11, s11, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10-NEXT: v_cmp_neq_f16_e64 s18, s16, s17
; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: v_cmp_u_f16_e64 s16, s16, s17
; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s14, 0
+; GFX10-NEXT: s_cselect_b32 s14, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s15, 0
+; GFX10-NEXT: s_cselect_b32 s15, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s18, 0
+; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s16, 0
+; GFX10-NEXT: s_cselect_b32 s16, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
-; GFX10-NEXT: v_cmp_lg_f16_e64 s4, s16, s17
-; GFX10-NEXT: s_addc_u32 s5, s5, 0
-; GFX10-NEXT: s_cmp_lg_u32 s7, 0
-; GFX10-NEXT: v_cmp_nlg_f16_e64 s7, s16, s17
-; GFX10-NEXT: s_addc_u32 s5, s5, s6
-; GFX10-NEXT: s_cmp_lg_u32 s4, 0
-; GFX10-NEXT: v_cmp_nlt_f16_e64 s4, s16, s17
-; GFX10-NEXT: s_addc_u32 s5, s5, s8
-; GFX10-NEXT: s_cmp_lg_u32 s7, 0
-; GFX10-NEXT: v_cmp_ngt_f16_e64 s6, s16, s17
-; GFX10-NEXT: s_addc_u32 s5, s5, s9
-; GFX10-NEXT: s_cmp_lg_u32 s4, 0
-; GFX10-NEXT: v_cmp_u_f16_e64 s4, s16, s17
-; GFX10-NEXT: s_addc_u32 s5, s5, s10
+; GFX10-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s6, 0
-; GFX10-NEXT: s_addc_u32 s5, s5, s11
-; GFX10-NEXT: s_cmp_lg_u32 s4, 0
-; GFX10-NEXT: s_addc_u32 s4, s5, s12
+; GFX10-NEXT: s_cselect_b32 s6, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s7, 0
+; GFX10-NEXT: s_cselect_b32 s7, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: s_cselect_b32 s8, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s10, 0
+; GFX10-NEXT: s_cselect_b32 s10, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10-NEXT: s_cselect_b32 s11, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s14, 0
+; GFX10-NEXT: s_cselect_b32 s14, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s15, 0
+; GFX10-NEXT: s_cselect_b32 s15, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s17, 0
+; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s16, 0
+; GFX10-NEXT: s_cselect_b32 s16, 1, 0
+; GFX10-NEXT: s_add_i32 s4, s4, s5
+; GFX10-NEXT: s_add_i32 s4, s4, s6
+; GFX10-NEXT: s_add_i32 s4, s4, s7
+; GFX10-NEXT: s_add_i32 s4, s4, s8
+; GFX10-NEXT: s_add_i32 s4, s4, s9
+; GFX10-NEXT: s_add_i32 s4, s4, s10
+; GFX10-NEXT: s_add_i32 s4, s4, s11
+; GFX10-NEXT: s_add_i32 s4, s4, s12
+; GFX10-NEXT: s_add_i32 s4, s4, s13
+; GFX10-NEXT: s_add_i32 s4, s4, s14
+; GFX10-NEXT: s_add_i32 s4, s4, s15
+; GFX10-NEXT: s_add_i32 s4, s4, s17
+; GFX10-NEXT: s_add_i32 s4, s4, s16
; GFX10-NEXT: v_mov_b32_e32 v2, s4
; GFX10-NEXT: flat_store_dword v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -61,54 +102,88 @@ define void @fcmp_f16_uniform(half inreg %a, half inreg %b, ptr %p) {
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s2, -1, 0
+; GFX12-NEXT: s_cselect_b32 s2, 1, 0
+; GFX12-NEXT: s_cmp_gt_f16 s0, s1
+; GFX12-NEXT: s_cselect_b32 s3, 1, 0
+; GFX12-NEXT: s_cmp_ge_f16 s0, s1
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
; GFX12-NEXT: s_cmp_lt_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s3, -1, 0
+; GFX12-NEXT: s_cselect_b32 s5, 1, 0
+; GFX12-NEXT: s_cmp_le_f16 s0, s1
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-NEXT: s_cmp_lg_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s4, -1, 0
+; GFX12-NEXT: s_cselect_b32 s7, 1, 0
+; GFX12-NEXT: s_cmp_o_f16 s0, s1
+; GFX12-NEXT: s_cselect_b32 s8, 1, 0
; GFX12-NEXT: s_cmp_nlg_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s5, -1, 0
-; GFX12-NEXT: s_cmp_nlt_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_cmp_ngt_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s7, -1, 0
-; GFX12-NEXT: s_cmp_u_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s8, -1, 0
-; GFX12-NEXT: s_cmp_gt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s9, 1, 0
-; GFX12-NEXT: s_cmp_ge_f16 s0, s1
+; GFX12-NEXT: s_cmp_nle_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s10, 1, 0
-; GFX12-NEXT: s_cmp_le_f16 s0, s1
+; GFX12-NEXT: s_cmp_nlt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s11, 1, 0
-; GFX12-NEXT: s_cmp_o_f16 s0, s1
+; GFX12-NEXT: s_cmp_nge_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s12, 1, 0
-; GFX12-NEXT: s_cmp_nle_f16 s0, s1
+; GFX12-NEXT: s_cmp_ngt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s13, 1, 0
-; GFX12-NEXT: s_cmp_nge_f16 s0, s1
-; GFX12-NEXT: s_cselect_b32 s14, 1, 0
; GFX12-NEXT: s_cmp_neq_f16 s0, s1
+; GFX12-NEXT: s_cselect_b32 s14, 1, 0
+; GFX12-NEXT: s_cmp_u_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s0, 1, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_cmp_lg_u32 s2, 0
-; GFX12-NEXT: s_add_co_ci_u32 s1, s9, 0
+; GFX12-NEXT: s_cselect_b32 s1, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s3, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s10
+; GFX12-NEXT: s_cselect_b32 s2, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s4, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s11
+; GFX12-NEXT: s_cselect_b32 s3, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s5, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s12
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s6, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s13
+; GFX12-NEXT: s_cselect_b32 s5, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s7, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s14
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s8, 0
+; GFX12-NEXT: s_cselect_b32 s7, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s9, 0
+; GFX12-NEXT: s_cselect_b32 s8, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s10, 0
+; GFX12-NEXT: s_cselect_b32 s9, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s11, 0
+; GFX12-NEXT: s_cselect_b32 s10, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s12, 0
+; GFX12-NEXT: s_cselect_b32 s11, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s13, 0
+; GFX12-NEXT: s_cselect_b32 s12, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s14, 0
+; GFX12-NEXT: s_cselect_b32 s13, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, 1, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s7
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s8
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s9
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s10
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s11
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s12
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_ci_u32 s0, s1, s0
+; GFX12-NEXT: s_add_co_i32 s1, s1, s13
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s0, s1, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: flat_store_b32 v[0:1], v2
@@ -163,34 +238,41 @@ define void @fcmp_f16_divergent(half %a, half %b, ptr %p) {
; GFX10-LABEL: fcmp_f16_divergent:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX10-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_le_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v4, v4, v5
; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v5, vcc_lo
-; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_le_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_add3_u32 v4, v4, v6, v7
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo
-; GFX10-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_add3_u32 v4, v4, v5, v8
; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v5, vcc_lo
-; GFX10-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_add3_u32 v4, v4, v6, v7
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo
-; GFX10-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_add3_u32 v4, v4, v5, v8
; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v5, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v0, vcc_lo, v4, v6, vcc_lo
+; GFX10-NEXT: v_add3_u32 v1, v4, v6, v7
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_add3_u32 v0, v1, v5, v0
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
index fb6112d9b6aa6..404a28331bb91 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
@@ -57,22 +57,42 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_fdiv_f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GFX8-NEXT: v_rcp_f32_e32 v4, v2
-; GFX8-NEXT: v_mul_f32_e32 v5, v3, v4
-; GFX8-NEXT: v_mad_f32 v6, -v2, v5, v3
-; GFX8-NEXT: v_mac_f32_e32 v5, v6, v4
-; GFX8-NEXT: v_mad_f32 v2, -v2, v5, v3
-; GFX8-NEXT: v_mul_f32_e32 v2, v2, v4
-; GFX8-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX8-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX8-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX8-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_fdiv_f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_fdiv_f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v3, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v3
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v3
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_fdiv_f16:
; GFX9-IEEE: ; %bb.0:
@@ -81,9 +101,12 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v2
; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
-; GFX9-IEEE-NEXT: v_mad_f32 v6, -v2, v5, v3
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v5, v6, v4
-; GFX9-IEEE-NEXT: v_mad_f32 v2, -v2, v5, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
@@ -107,22 +130,41 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_fdiv_f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v5, v0
-; GFX10-NEXT: v_rcp_f32_e32 v3, v2
-; GFX10-NEXT: v_fma_mix_f32 v4, v0, v3, neg(0) op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_mad_f32 v6, -v2, v4, v5
-; GFX10-NEXT: v_mac_f32_e32 v4, v6, v3
-; GFX10-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX10-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX10-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-IEEE-LABEL: v_fdiv_f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v3, v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
+; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_fdiv_f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v4
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_f16:
; GFX11: ; %bb.0:
@@ -222,22 +264,42 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_fdiv_f16_ulp25:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GFX8-NEXT: v_rcp_f32_e32 v4, v2
-; GFX8-NEXT: v_mul_f32_e32 v5, v3, v4
-; GFX8-NEXT: v_mad_f32 v6, -v2, v5, v3
-; GFX8-NEXT: v_mac_f32_e32 v5, v6, v4
-; GFX8-NEXT: v_mad_f32 v2, -v2, v5, v3
-; GFX8-NEXT: v_mul_f32_e32 v2, v2, v4
-; GFX8-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX8-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX8-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX8-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_fdiv_f16_ulp25:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_fdiv_f16_ulp25:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v3, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v3
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v3
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_fdiv_f16_ulp25:
; GFX9-IEEE: ; %bb.0:
@@ -246,9 +308,12 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v2
; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
-; GFX9-IEEE-NEXT: v_mad_f32 v6, -v2, v5, v3
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v5, v6, v4
-; GFX9-IEEE-NEXT: v_mad_f32 v2, -v2, v5, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
@@ -272,22 +337,41 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_fdiv_f16_ulp25:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v5, v0
-; GFX10-NEXT: v_rcp_f32_e32 v3, v2
-; GFX10-NEXT: v_fma_mix_f32 v4, v0, v3, neg(0) op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_mad_f32 v6, -v2, v4, v5
-; GFX10-NEXT: v_mac_f32_e32 v4, v6, v3
-; GFX10-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX10-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX10-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-IEEE-LABEL: v_fdiv_f16_ulp25:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v3, v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
+; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_fdiv_f16_ulp25:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v4
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_f16_ulp25:
; GFX11: ; %bb.0:
@@ -2183,7 +2267,9 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -2198,11 +2284,12 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
;
; GFX6-FLUSH-LABEL: s_fdiv_f16:
; GFX6-FLUSH: ; %bb.0:
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v0, v0, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s1
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, v1, v0, v1
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
@@ -2211,30 +2298,54 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, v1
+; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v1, v0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-FLUSH-NEXT: ; return to shader part epilog
;
-; GFX8-LABEL: s_fdiv_f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, s1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v1, s0
-; GFX8-NEXT: v_rcp_f32_e32 v2, v0
-; GFX8-NEXT: v_mul_f32_e32 v3, v1, v2
-; GFX8-NEXT: v_mad_f32 v4, -v0, v3, v1
-; GFX8-NEXT: v_mac_f32_e32 v3, v4, v2
-; GFX8-NEXT: v_mad_f32 v0, -v0, v3, v1
-; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX8-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
-; GFX8-NEXT: v_div_fixup_f16 v0, v0, s1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-IEEE-LABEL: s_fdiv_f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s0
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v2, v0
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v1, v2
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v4, -v0, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v4, v4, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v0, -v0, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-IEEE-NEXT: s_and_b32 s2, s2, 0xff800000
+; GFX8-IEEE-NEXT: v_add_f32_e32 v0, s2, v3
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v0, v1, s0
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX8-FLUSH-LABEL: s_fdiv_f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v2, v0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v1, v2
+; GFX8-FLUSH-NEXT: v_mad_f32 v4, -v0, v3, v1
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v3, v4, v2
+; GFX8-FLUSH-NEXT: v_mad_f32 v0, -v0, v3, v1
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-FLUSH-NEXT: s_and_b32 s2, s2, 0xff800000
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v0, s2, v3
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v1, s0
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-FLUSH-NEXT: ; return to shader part epilog
;
; GFX9-IEEE-LABEL: s_fdiv_f16:
; GFX9-IEEE: ; %bb.0:
@@ -2242,51 +2353,77 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX9-IEEE-NEXT: v_rcp_f32_e32 v2, v0
; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v1, v2
-; GFX9-IEEE-NEXT: v_mad_f32 v4, -v0, v3, v1
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v3, v4, v2
-; GFX9-IEEE-NEXT: v_mad_f32 v0, -v0, v3, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v4, -v0, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v4, v4, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v0, -v0, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-IEEE-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX9-IEEE-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX9-IEEE-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-IEEE-NEXT: s_and_b32 s2, s2, 0xff800000
+; GFX9-IEEE-NEXT: v_add_f32_e32 v0, s2, v3
; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-IEEE-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v0, s1, v1
+; GFX9-IEEE-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v0, v1, s0
; GFX9-IEEE-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-IEEE-NEXT: ; return to shader part epilog
;
; GFX9-FLUSH-LABEL: s_fdiv_f16:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
-; GFX9-FLUSH-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v1, s1
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v1, s0, v0, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -s1, v1, v2 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v1, v3, v0
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -s1, v1, v2 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v2, s0, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -v1, v2, s0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v2, v3, v0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -v1, v2, s0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v3, v0
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, s1, v2
; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, v2
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v1, s0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-FLUSH-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: s_fdiv_f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_cvt_f32_f16_e32 v0, s1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v3, s0
-; GFX10-NEXT: v_rcp_f32_e32 v1, v0
-; GFX10-NEXT: v_fma_mix_f32 v2, s0, v1, neg(0) op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_mad_f32 v4, -v0, v2, v3
-; GFX10-NEXT: v_mac_f32_e32 v2, v4, v1
-; GFX10-NEXT: v_mad_f32 v0, -v0, v2, v3
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_div_fixup_f16 v0, v0, s1, s0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
+; GFX10-IEEE-LABEL: s_fdiv_f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v1, s0, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, -s1, v1, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, s0, 1.0, v2 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v2, v0
+; GFX10-IEEE-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, -s1, v1, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, s0, 1.0, v2 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v2, v0
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-IEEE-NEXT: s_and_b32 s2, s2, 0xff800000
+; GFX10-IEEE-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, s1, s0
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX10-FLUSH-LABEL: s_fdiv_f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, s0
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v1, v0
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v3, s0, v1, neg(0) op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_mad_f32 v4, -v0, v3, v2
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v3, v4, v1
+; GFX10-FLUSH-NEXT: v_mad_f32 v0, -v0, v3, v2
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-FLUSH-NEXT: s_and_b32 s2, s2, 0xff800000
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v0, s2, v3
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, s0
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-FLUSH-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fdiv_f16:
; GFX11: ; %bb.0:
@@ -2298,8 +2435,9 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX11-NEXT: v_fmac_f32_e32 v1, v2, v0
; GFX11-NEXT: v_fma_mix_f32 v2, -s1, v1, s0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_mul_f32_e32 v0, v2, v0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff800000
+; GFX11-NEXT: v_add_f32_e32 v0, s2, v1
; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX11-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -2318,7 +2456,9 @@ define amdgpu_ps i16 @s_fdiv_f16_arcp(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -2333,11 +2473,12 @@ define amdgpu_ps i16 @s_fdiv_f16_arcp(i16 inreg %a.arg, i16 inreg %b.arg) {
;
; GFX6-FLUSH-LABEL: s_fdiv_f16_arcp:
; GFX6-FLUSH: ; %bb.0:
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v0, v0, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s1
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, v1, v0, v1
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
@@ -2346,8 +2487,9 @@ define amdgpu_ps i16 @s_fdiv_f16_arcp(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, v1
+; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v1, v0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
@@ -2674,7 +2816,9 @@ define amdgpu_ps i16 @s_rcp_f16(i16 inreg %a.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-IEEE-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v2, v1
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v2, v4, v2, v2
; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v3, v2
@@ -2691,8 +2835,9 @@ define amdgpu_ps i16 @s_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v2, v1
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, vcc, 1.0, v0, 1.0
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v2, v4, v2, v2
@@ -2701,6 +2846,7 @@ define amdgpu_ps i16 @s_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v4, v5, v2, v4
; GFX6-FLUSH-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -2738,7 +2884,9 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-IEEE-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, -1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v2, v1
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, -1.0, v0, -1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], -1.0, v0, -1.0
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v2, v4, v2, v2
; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v3, v2
@@ -2755,8 +2903,9 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, -1.0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[0:1], -1.0, v0, -1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v2, v1
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, vcc, -1.0, v0, -1.0
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v2, v4, v2, v2
@@ -2765,6 +2914,7 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v4, v5, v2, v4
; GFX6-FLUSH-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v1, v0, -1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -2797,13 +2947,54 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
}
define amdgpu_ps i16 @s_rsq_f16(i16 inreg %a.arg) {
-; GFX6-LABEL: s_rsq_f16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX6-NEXT: v_rsq_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: ; return to shader part epilog
+; GFX6-IEEE-LABEL: s_rsq_f16:
+; GFX6-IEEE: ; %bb.0:
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v0, v0
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v2, v1
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-IEEE-NEXT: v_fma_f32 v4, -v1, v2, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v2, v4, v2, v2
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v3, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v5, -v1, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v5, v2, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v1, -v1, v4, v3
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v1, v1, v2, v4
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX6-FLUSH-LABEL: s_rsq_f16:
+; GFX6-FLUSH: ; %bb.0:
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
+; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v2, v1
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v1, v2, 1.0
+; GFX6-FLUSH-NEXT: v_fma_f32 v2, v4, v2, v2
+; GFX6-FLUSH-NEXT: v_mul_f32_e32 v4, v3, v2
+; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v1, v4, v3
+; GFX6-FLUSH-NEXT: v_fma_f32 v4, v5, v2, v4
+; GFX6-FLUSH-NEXT: v_fma_f32 v1, -v1, v4, v3
+; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-FLUSH-NEXT: v_div_fmas_f32 v1, v1, v2, v4
+; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
+; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-FLUSH-NEXT: ; return to shader part epilog
;
; GFX89-LABEL: s_rsq_f16:
; GFX89: ; %bb.0:
@@ -3074,23 +3265,23 @@ define half @v_neg_rsq_f16(half %a) {
; GFX89-LABEL: v_neg_rsq_f16:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX89-NEXT: v_rsq_f16_e32 v0, v0
-; GFX89-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX89-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX89-NEXT: v_rcp_f16_e64 v0, -v0
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_neg_rsq_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rsq_f16_e32 v0, v0
-; GFX10-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX10-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX10-NEXT: v_rcp_f16_e64 v0, -v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rsq_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_rsq_f16_e32 v0, v0
+; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX11-NEXT: v_rcp_f16_e64 v0, -v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract half @llvm.sqrt.f16(half %a)
%fdiv = fdiv contract half -1.0, %sqrt
@@ -3146,23 +3337,26 @@ define { half, half } @v_rsq_f16_multi_use(half %a) {
; GFX89-LABEL: v_rsq_f16_multi_use:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX89-NEXT: v_sqrt_f16_e32 v0, v0
-; GFX89-NEXT: v_rcp_f16_e32 v1, v0
+; GFX89-NEXT: v_sqrt_f16_e32 v2, v0
+; GFX89-NEXT: v_rsq_f16_e32 v1, v0
+; GFX89-NEXT: v_mov_b32_e32 v0, v2
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_rsq_f16_multi_use:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_sqrt_f16_e32 v0, v0
-; GFX10-NEXT: v_rcp_f16_e32 v1, v0
+; GFX10-NEXT: v_sqrt_f16_e32 v2, v0
+; GFX10-NEXT: v_rsq_f16_e32 v1, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rsq_f16_multi_use:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX11-NEXT: v_sqrt_f16_e32 v2, v0
+; GFX11-NEXT: v_rsq_f16_e32 v1, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_rcp_f16_e32 v1, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract half @llvm.sqrt.f16(half %a)
%insert.0 = insertvalue { half, half } poison, half %sqrt, 0
@@ -3508,23 +3702,23 @@ define half @v_neg_rsq_f16_fabs(half %a) {
; GFX89-LABEL: v_neg_rsq_f16_fabs:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX89-NEXT: v_rsq_f16_e64 v0, |v0|
-; GFX89-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX89-NEXT: v_sqrt_f16_e64 v0, |v0|
+; GFX89-NEXT: v_rcp_f16_e64 v0, -v0
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_neg_rsq_f16_fabs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rsq_f16_e64 v0, |v0|
-; GFX10-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX10-NEXT: v_sqrt_f16_e64 v0, |v0|
+; GFX10-NEXT: v_rcp_f16_e64 v0, -v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rsq_f16_fabs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_rsq_f16_e64 v0, |v0|
+; GFX11-NEXT: v_sqrt_f16_e64 v0, |v0|
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX11-NEXT: v_rcp_f16_e64 v0, -v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%a.fabs = call half @llvm.fabs.f16(half %a)
%sqrt = call contract half @llvm.sqrt.f16(half %a.fabs)
@@ -3643,23 +3837,23 @@ define half @v_neg_rsq_f16_arcp(half %a) {
; GFX89-LABEL: v_neg_rsq_f16_arcp:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX89-NEXT: v_rsq_f16_e32 v0, v0
-; GFX89-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX89-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX89-NEXT: v_rcp_f16_e64 v0, -v0
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_neg_rsq_f16_arcp:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rsq_f16_e32 v0, v0
-; GFX10-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX10-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX10-NEXT: v_rcp_f16_e64 v0, -v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rsq_f16_arcp:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_rsq_f16_e32 v0, v0
+; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX11-NEXT: v_rcp_f16_e64 v0, -v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract half @llvm.sqrt.f16(half %a)
%fdiv = fdiv contract arcp half -1.0, %sqrt
@@ -3991,9 +4185,5 @@ declare <2 x half> @llvm.fabs.v2f16(<2 x half>)
declare <2 x half> @llvm.sqrt.v2f16(<2 x half>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10-FLUSH: {{.*}}
-; GFX10-IEEE: {{.*}}
; GFX11-FLUSH: {{.*}}
; GFX11-IEEE: {{.*}}
-; GFX8-FLUSH: {{.*}}
-; GFX8-IEEE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
index 71192fe57372b..c1785c0f62b12 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=FIJI %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 < %s | FileCheck -check-prefix=FIJI %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GFX12 %s
define float @ffloor_s(float inreg %val) {
; FIJI-LABEL: ffloor_s:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
index f97b0a8120d3d..6d1275165a4c9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
@@ -4,9 +4,9 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.0a < %s | FileCheck -check-prefix=GFX90A %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
define float @v_fma_f32(float %x, float %y, float %z) {
; GFX6-LABEL: v_fma_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmad.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmad.ll
index eaa1319f166b6..82a7b459ec035 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmad.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX10 %s
define amdgpu_ps float @fmad_s32_uniform(float inreg %a, float inreg %b, float inreg %c) {
; GFX8-LABEL: fmad_s32_uniform:
@@ -51,16 +51,16 @@ define amdgpu_ps float @fmad_s32_div(float %a, float %b, float %c) {
define amdgpu_ps half @fmad_s16_uniform(half inreg %a, half inreg %b, half inreg %c) {
; GFX8-LABEL: fmad_s16_uniform:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mad_f16 v0, s0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: v_mad_f16 v0, s0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: fmad_s16_uniform:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mad_legacy_f16 v0, s0, v1, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mad_legacy_f16 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: fmad_s16_uniform:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmamix-constant-bus-violation.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmamix-constant-bus-violation.ll
index 8e61722bc2e66..37be9cb9c9f37 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmamix-constant-bus-violation.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmamix-constant-bus-violation.ll
@@ -1,13 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.08 %s -o - | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu9.08 %s -o - | FileCheck %s
define float @test_fmamix_constant_bus_violation_sss(i32 inreg %val.0, i32 inreg %val.1, i32 inreg %val.2) #0 {
; CHECK-LABEL: test_fmamix_constant_bus_violation_sss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v0, s17
-; CHECK-NEXT: v_mov_b32_e32 v1, s18
-; CHECK-NEXT: v_fma_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s6, s18, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v0, s5
+; CHECK-NEXT: v_mov_b32_e32 v1, s6
+; CHECK-NEXT: v_fma_mix_f32 v0, s4, v0, v1 op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@@ -29,8 +32,10 @@ define float @test_fmamix_constant_bus_violation_ssv(i32 inreg %val.0, i32 inreg
; CHECK-LABEL: test_fmamix_constant_bus_violation_ssv:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
-; CHECK-NEXT: v_fma_mix_f32 v0, s16, v1, v0 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_fma_mix_f32 v0, s4, v1, v0 op_sel:[0,0,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@@ -52,8 +57,10 @@ define float @test_fmamix_constant_bus_violation_svs(i32 inreg %val.0, i32 %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_svs:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
-; CHECK-NEXT: v_fma_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_fma_mix_f32 v0, s4, v0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@@ -75,8 +82,10 @@ define float @test_fmamix_constant_bus_violation_vss(i32 %val.0, i32 inreg %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_vss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
-; CHECK-NEXT: v_fma_mix_f32 v0, v0, s16, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_fma_mix_f32 v0, v0, s4, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
index b05621558b04c..aa51714013889 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
@@ -247,14 +247,13 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX10-LABEL: test_max_K0min_ValK1_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_min_f16_e32 v0, 4.0, v0
-; GFX10-NEXT: v_max_f16_e32 v0, 2.0, v0
+; GFX10-NEXT: v_med3_f16 v0, v0, 2.0, 4.0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_max_K0min_ValK1_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
+; GFX1170-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_max_K0min_ValK1_f16:
@@ -274,7 +273,7 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
+; GFX12-FAKE16-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%minnum = call nnan half @llvm.minnum.f16(half %a, half 4.0)
%fmed = call nnan half @llvm.maxnum.f16(half 2.0, half %minnum)
@@ -292,14 +291,13 @@ define half @test_max_K0min_K1Val_f16(half %a) #1 {
; GFX10-LABEL: test_max_K0min_K1Val_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_min_f16_e32 v0, 4.0, v0
-; GFX10-NEXT: v_max_f16_e32 v0, 2.0, v0
+; GFX10-NEXT: v_med3_f16 v0, v0, 2.0, 4.0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_max_K0min_K1Val_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
+; GFX1170-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_max_K0min_K1Val_f16:
@@ -319,7 +317,7 @@ define half @test_max_K0min_K1Val_f16(half %a) #1 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
+; GFX12-FAKE16-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%minnum = call nnan half @llvm.minnum.f16(half 4.0, half %a)
%fmed = call nnan half @llvm.maxnum.f16(half 2.0, half %minnum)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
index 4444102abbc74..0baba45e541a1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps half @fmed3_s16_uniform(half inreg %a, half inreg %b, half inreg %c) {
; GFX9-LABEL: fmed3_s16_uniform:
@@ -34,7 +34,11 @@ define amdgpu_ps half @fmed3_s16_uniform_salu_use(half inreg %a, half inreg %b,
; GFX12-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_med3_num_f16 v0.l, s0, s1, v0.l
-; GFX12-NEXT: v_add_f16_e32 v0.l, s3, v0.l
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: s_add_f16 s0, s0, s3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
%result = call half @llvm.amdgcn.fmed3.f16(half %a, half %b, half %c)
%add = fadd half %result, %d
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmin3-fmax3-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmin3-fmax3-combine.ll
index 1c02eb0fedad5..7b5d1b02c07df 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmin3-fmax3-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmin3-fmax3-combine.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
define float @test_fmin3(float %a, float %b, float %c) {
; GFX10-LABEL: test_fmin3:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
index 5571e08d78ea5..a470767e80a39 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
define amdgpu_ps half @fmul_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-FAKE16-LABEL: fmul_s16_uniform:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
index 1bfd6fbee462b..0720794f94102 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
define amdgpu_ps void @v_fneg_f16(half %in, ptr addrspace(1) %out) {
; GCN-LABEL: v_fneg_f16:
@@ -13,27 +13,44 @@ define amdgpu_ps void @v_fneg_f16(half %in, ptr addrspace(1) %out) {
ret void
}
define amdgpu_ps void @s_fneg_f16(half inreg %in, ptr addrspace(1) %out) {
-; GCN-LABEL: s_fneg_f16:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_xor_b32 s0, s0, 0x8000
-; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: global_store_b16 v[0:1], v2, off
-; GCN-NEXT: s_endpgm
+; GFX11-LABEL: s_fneg_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b32_e64 v2, 0x8000, s0
+; GFX11-NEXT: global_store_b16 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: s_fneg_f16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%fneg = fneg half %in
store half %fneg, ptr addrspace(1) %out
ret void
}
define amdgpu_ps void @s_fneg_f16_salu_use(half inreg %in, i32 inreg %val, ptr addrspace(1) %out) {
-; GCN-LABEL: s_fneg_f16_salu_use:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_xor_b32 s0, s0, 0x8000
-; GCN-NEXT: s_cmp_eq_u32 s1, 0
-; GCN-NEXT: s_cselect_b32 s1, -1, 0
-; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, s0, s1
-; GCN-NEXT: global_store_b16 v[0:1], v2, off
-; GCN-NEXT: s_endpgm
+; GFX11-LABEL: s_fneg_f16_salu_use:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b32_e64 v2, 0x8000, s0
+; GFX11-NEXT: s_cmp_eq_u32 s1, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: s_cselect_b32 s0, s0, 0
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-NEXT: global_store_b16 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: s_fneg_f16_salu_use:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX12-NEXT: s_cmp_eq_u32 s1, 0
+; GFX12-NEXT: s_cselect_b32 s0, s0, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%fneg = fneg half %in
%cond = icmp eq i32 %val, 0
%sel = select i1 %cond, half %fneg, half 0.0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
index 7be6cedc82ae3..50ec9e197ca4a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
@@ -135,34 +135,34 @@ define half @v_pow_f16(half %x, half %y) {
; GFX8-LABEL: v_pow_f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT: v_log_f16_e32 v0, v0
; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX8-NEXT: v_log_f32_e32 v0, v0
-; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX8-NEXT: v_exp_f32_e32 v0, v0
+; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT: v_exp_f16_e32 v0, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_pow_f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-NEXT: v_log_f16_e32 v0, v0
; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9-NEXT: v_log_f32_e32 v0, v0
-; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX9-NEXT: v_exp_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT: v_exp_f16_e32 v0, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_pow_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX10-NEXT: v_log_f16_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX10-NEXT: v_log_f32_e32 v0, v0
-; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX10-NEXT: v_exp_f32_e32 v0, v0
+; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-NEXT: v_exp_f16_e32 v0, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_f16:
@@ -182,15 +182,15 @@ define half @v_pow_f16(half %x, half %y) {
; GFX11-FAKE16-LABEL: v_pow_f16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%pow = call half @llvm.pow.f16(half %x, half %y)
ret half %pow
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
index ec7c51e33126f..b0f5431bd5452 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
@@ -1,110 +1,126 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; UNSUPPORTED: true
; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgpu7.04 < %s | FileCheck --check-prefix=CI %s
; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgpu8.02 < %s | FileCheck --check-prefix=VI %s
define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1, ptr addrspace(1) %in2) #0 {
; CI-LABEL: frem_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; CI-NEXT: s_mov_b32 s7, 0xf000
-; CI-NEXT: s_mov_b32 s6, -1
-; CI-NEXT: s_mov_b32 s2, s6
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_load_dword s2, s[2:3], 0x0
+; CI-NEXT: s_load_dword s3, s[4:5], 0x2
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s4, s10
-; CI-NEXT: s_mov_b32 s5, s11
-; CI-NEXT: s_mov_b32 s3, s7
-; CI-NEXT: buffer_load_ushort v0, off, s[4:7], 0
-; CI-NEXT: buffer_load_ushort v1, off, s[0:3], 0 offset:8
-; CI-NEXT: s_waitcnt vmcnt(1)
-; CI-NEXT: v_cvt_f32_f16_e64 v4, |v0|
-; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e64 v3, |v1|
-; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v4, v3
-; CI-NEXT: s_cbranch_vccz .LBB0_2
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s3|
+; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[4:5], vcc, vcc
+; CI-NEXT: s_mov_b32 s5, 1
+; CI-NEXT: ; implicit-def: $sgpr4
+; CI-NEXT: s_cbranch_scc0 .LBB0_2
; CI-NEXT: ; %bb.1: ; %frem.else
-; CI-NEXT: v_and_b32_e32 v2, 0x8000, v0
-; CI-NEXT: v_cmp_eq_f32_e32 vcc, v4, v3
-; CI-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
-; CI-NEXT: s_mov_b64 s[0:1], 0
-; CI-NEXT: s_branch .LBB0_3
-; CI-NEXT: .LBB0_2:
-; CI-NEXT: s_mov_b64 s[0:1], -1
-; CI-NEXT: ; implicit-def: $vgpr2
-; CI-NEXT: .LBB0_3: ; %Flow18
-; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: s_cselect_b32 s0, 1, 0
-; CI-NEXT: s_cmp_lg_u32 s0, 1
-; CI-NEXT: s_cbranch_scc1 .LBB0_9
-; CI-NEXT: ; %bb.4: ; %frem.compute
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v7, v4
-; CI-NEXT: v_frexp_mant_f32_e32 v2, v4
-; CI-NEXT: v_frexp_mant_f32_e32 v4, v3
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v8, v3
-; CI-NEXT: v_ldexp_f32_e64 v3, v4, 1
-; CI-NEXT: v_div_scale_f32 v9, s[0:1], v3, v3, 1.0
-; CI-NEXT: v_ldexp_f32_e64 v5, v2, 11
-; CI-NEXT: v_add_i32_e32 v2, vcc, -1, v8
-; CI-NEXT: v_not_b32_e32 v4, v2
-; CI-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; CI-NEXT: v_div_scale_f32 v6, vcc, 1.0, v3, 1.0
-; CI-NEXT: v_rcp_f32_e32 v10, v9
+; CI-NEXT: s_and_b32 s4, s2, 0xffff8000
+; CI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[6:7], vcc, vcc
+; CI-NEXT: s_mov_b32 s5, 0
+; CI-NEXT: s_cselect_b32 s4, s4, s2
+; CI-NEXT: .LBB0_2: ; %Flow18
+; CI-NEXT: s_xor_b32 s5, s5, 1
+; CI-NEXT: s_cmp_lg_u32 s5, 0
+; CI-NEXT: s_cbranch_scc1 .LBB0_8
+; CI-NEXT: ; %bb.3: ; %frem.compute
+; CI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; CI-NEXT: v_readfirstlane_b32 s7, v1
+; CI-NEXT: v_ldexp_f32_e64 v1, v2, 11
+; CI-NEXT: v_readfirstlane_b32 s6, v1
+; CI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; CI-NEXT: v_readfirstlane_b32 s8, v0
+; CI-NEXT: v_ldexp_f32_e64 v0, v1, 1
+; CI-NEXT: v_div_scale_f32 v1, s[10:11], v0, v0, 1.0
+; CI-NEXT: s_add_i32 s5, s7, -1
+; CI-NEXT: s_add_i32 s4, s8, -1
+; CI-NEXT: s_sub_i32 s5, s5, s4
+; CI-NEXT: v_div_scale_f32 v2, s[10:11], 1.0, v0, 1.0
+; CI-NEXT: s_or_b64 s[10:11], s[10:11], s[10:11]
+; CI-NEXT: v_rcp_f32_e32 v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v11, -v9, v10, 1.0
-; CI-NEXT: v_fma_f32 v10, v11, v10, v10
-; CI-NEXT: v_mul_f32_e32 v11, v6, v10
-; CI-NEXT: v_fma_f32 v12, -v9, v11, v6
-; CI-NEXT: v_fma_f32 v11, v12, v10, v11
-; CI-NEXT: v_fma_f32 v6, -v9, v11, v6
+; CI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; CI-NEXT: v_fma_f32 v3, v4, v3, v3
+; CI-NEXT: v_mul_f32_e32 v4, v2, v3
+; CI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; CI-NEXT: v_fma_f32 v4, v5, v3, v4
+; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v6, v6, v10, v11
-; CI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v4
-; CI-NEXT: v_div_fixup_f32 v6, v6, v3, 1.0
-; CI-NEXT: s_cbranch_vccnz .LBB0_8
-; CI-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; CI-NEXT: v_sub_i32_e32 v4, vcc, v7, v8
-; CI-NEXT: v_add_i32_e32 v4, vcc, 11, v4
-; CI-NEXT: .LBB0_6: ; %frem.loop_body
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_cmp_le_i32 s5, 11
+; CI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; CI-NEXT: s_cbranch_scc1 .LBB0_6
+; CI-NEXT: ; %bb.4: ; %frem.loop_body.preheader
+; CI-NEXT: s_add_i32 s7, s7, 11
+; CI-NEXT: s_sub_i32 s5, s7, s8
+; CI-NEXT: .LBB0_5: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: v_mov_b32_e32 v7, v5
-; CI-NEXT: v_mul_f32_e32 v5, v7, v6
-; CI-NEXT: v_rndne_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v5, -v5, v3, v7
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v5
-; CI-NEXT: v_add_f32_e32 v8, v5, v3
-; CI-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc
-; CI-NEXT: v_add_i32_e32 v4, vcc, -11, v4
-; CI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v4
-; CI-NEXT: v_ldexp_f32_e64 v5, v5, 11
-; CI-NEXT: s_cbranch_vccnz .LBB0_6
-; CI-NEXT: ; %bb.7: ; %Flow
-; CI-NEXT: v_mov_b32_e32 v5, v7
-; CI-NEXT: .LBB0_8: ; %frem.loop_exit
-; CI-NEXT: v_add_i32_e32 v4, vcc, -10, v4
-; CI-NEXT: v_ldexp_f32_e32 v4, v5, v4
-; CI-NEXT: v_mul_f32_e32 v5, v4, v6
-; CI-NEXT: v_rndne_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v4, -v5, v3, v4
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v4
-; CI-NEXT: v_add_f32_e32 v3, v4, v3
-; CI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; CI-NEXT: v_ldexp_f32_e32 v2, v3, v2
-; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; CI-NEXT: v_and_b32_e32 v3, 0x8000, v0
-; CI-NEXT: v_and_b32_e32 v2, 0x7fff, v2
-; CI-NEXT: v_or_b32_e32 v2, v2, v3
-; CI-NEXT: .LBB0_9: ; %Flow19
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; CI-NEXT: s_mov_b32 s0, 0x7f800000
-; CI-NEXT: s_mov_b32 s11, 0xf000
-; CI-NEXT: v_cmp_lg_f32_e32 vcc, 0, v1
-; CI-NEXT: v_cmp_nle_f32_e64 s[0:1], s0, v0
-; CI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; CI-NEXT: v_mov_b32_e32 v0, 0x7e00
-; CI-NEXT: s_mov_b32 s10, -1
-; CI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; CI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; CI-NEXT: v_mul_f32_e32 v2, s6, v1
+; CI-NEXT: v_rndne_f32_e32 v2, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s6
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s7, v2
+; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v2, v2, v0
+; CI-NEXT: v_readfirstlane_b32 s8, v2
+; CI-NEXT: s_cselect_b32 s7, s8, s7
+; CI-NEXT: v_ldexp_f32_e64 v2, s7, 11
+; CI-NEXT: v_readfirstlane_b32 s8, v2
+; CI-NEXT: s_add_i32 s5, s5, -11
+; CI-NEXT: s_cmp_gt_i32 s5, 11
+; CI-NEXT: s_mov_b32 s7, s6
+; CI-NEXT: s_mov_b32 s6, s8
+; CI-NEXT: s_cbranch_scc1 .LBB0_5
+; CI-NEXT: s_branch .LBB0_7
+; CI-NEXT: .LBB0_6:
+; CI-NEXT: s_mov_b32 s7, s6
+; CI-NEXT: .LBB0_7: ; %frem.loop_exit
+; CI-NEXT: s_add_i32 s5, s5, -10
+; CI-NEXT: v_mov_b32_e32 v2, s5
+; CI-NEXT: v_ldexp_f32_e32 v2, s7, v2
+; CI-NEXT: v_mul_f32_e32 v1, v2, v1
+; CI-NEXT: v_rndne_f32_e32 v1, v1
+; CI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; CI-NEXT: s_or_b64 s[6:7], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v0, v1, v0
+; CI-NEXT: v_readfirstlane_b32 s5, v1
+; CI-NEXT: v_readfirstlane_b32 s6, v0
+; CI-NEXT: s_cselect_b32 s5, s6, s5
+; CI-NEXT: v_mov_b32_e32 v0, s4
+; CI-NEXT: v_ldexp_f32_e32 v0, s5, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CI-NEXT: s_and_b32 s5, s2, 0xffff8000
+; CI-NEXT: v_readfirstlane_b32 s4, v0
+; CI-NEXT: s_and_b32 s4, s4, 0x7fff
+; CI-NEXT: s_or_b32 s4, s4, s5
+; CI-NEXT: .LBB0_8: ; %Flow19
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; CI-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s2|
+; CI-NEXT: s_or_b64 s[6:7], vcc, vcc
+; CI-NEXT: s_cselect_b32 s5, 1, 0
+; CI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
+; CI-NEXT: s_or_b64 s[2:3], vcc, vcc
+; CI-NEXT: s_cselect_b32 s2, 1, 0
+; CI-NEXT: s_cmp_lg_u32 s5, 0
+; CI-NEXT: s_cselect_b32 s3, 0x7e00, s4
+; CI-NEXT: s_cmp_lg_u32 s2, 0
+; CI-NEXT: s_cselect_b32 s2, s3, 0x7e00
+; CI-NEXT: v_mov_b32_e32 v0, s2
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: buffer_store_short v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: frem_f16:
@@ -112,100 +128,116 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: s_add_u32 s2, s4, 8
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: s_addc_u32 s3, s5, 0
-; VI-NEXT: flat_load_ushort v0, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_load_ushort v1, v[1:2]
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_f16_e64 v4, |v0|
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e64 v3, |v1|
-; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v4, v3
-; VI-NEXT: s_cbranch_vccz .LBB0_2
+; VI-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-NEXT: s_load_dword s3, s[4:5], 0x8
+; VI-NEXT: s_mov_b32 s5, 1
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
+; VI-NEXT: v_cvt_f32_f16_e64 v0, |s3|
+; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_cbranch_scc0 .LBB0_2
; VI-NEXT: ; %bb.1: ; %frem.else
-; VI-NEXT: v_and_b32_e32 v2, 0x8000, v0
-; VI-NEXT: v_cmp_eq_f32_e32 vcc, v4, v3
-; VI-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
-; VI-NEXT: s_mov_b64 s[2:3], 0
-; VI-NEXT: s_branch .LBB0_3
-; VI-NEXT: .LBB0_2:
-; VI-NEXT: s_mov_b64 s[2:3], -1
-; VI-NEXT: ; implicit-def: $vgpr2
-; VI-NEXT: .LBB0_3: ; %Flow18
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cbranch_scc1 .LBB0_9
-; VI-NEXT: ; %bb.4: ; %frem.compute
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v7, v4
-; VI-NEXT: v_frexp_mant_f32_e32 v2, v4
-; VI-NEXT: v_frexp_mant_f32_e32 v4, v3
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v8, v3
-; VI-NEXT: v_ldexp_f32 v3, v4, 1
-; VI-NEXT: v_div_scale_f32 v9, s[2:3], v3, v3, 1.0
-; VI-NEXT: v_ldexp_f32 v5, v2, 11
-; VI-NEXT: v_add_u32_e32 v2, vcc, -1, v8
-; VI-NEXT: v_not_b32_e32 v4, v2
-; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v7
-; VI-NEXT: v_div_scale_f32 v6, vcc, 1.0, v3, 1.0
-; VI-NEXT: v_rcp_f32_e32 v10, v9
+; VI-NEXT: s_and_b32 s4, s2, 0xffff8000
+; VI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_cselect_b32 s4, s4, s2
+; VI-NEXT: .LBB0_2: ; %Flow18
+; VI-NEXT: s_xor_b32 s5, s5, 1
+; VI-NEXT: s_cmp_lg_u32 s5, 0
+; VI-NEXT: s_cbranch_scc1 .LBB0_8
+; VI-NEXT: ; %bb.3: ; %frem.compute
+; VI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; VI-NEXT: v_readfirstlane_b32 s7, v1
+; VI-NEXT: v_ldexp_f32 v1, v2, 11
+; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; VI-NEXT: v_readfirstlane_b32 s8, v0
+; VI-NEXT: v_ldexp_f32 v0, v1, 1
+; VI-NEXT: v_div_scale_f32 v1, s[10:11], v0, v0, 1.0
+; VI-NEXT: s_add_i32 s5, s7, -1
+; VI-NEXT: s_add_i32 s4, s8, -1
+; VI-NEXT: s_sub_i32 s5, s5, s4
+; VI-NEXT: v_div_scale_f32 v2, s[10:11], 1.0, v0, 1.0
+; VI-NEXT: s_cmp_lg_u64 s[10:11], 0
+; VI-NEXT: v_rcp_f32_e32 v3, v1
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; VI-NEXT: v_fma_f32 v11, -v9, v10, 1.0
-; VI-NEXT: v_fma_f32 v10, v11, v10, v10
-; VI-NEXT: v_mul_f32_e32 v11, v6, v10
-; VI-NEXT: v_fma_f32 v12, -v9, v11, v6
-; VI-NEXT: v_fma_f32 v11, v12, v10, v11
-; VI-NEXT: v_fma_f32 v6, -v9, v11, v6
+; VI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; VI-NEXT: v_fma_f32 v3, v4, v3, v3
+; VI-NEXT: v_mul_f32_e32 v4, v2, v3
+; VI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; VI-NEXT: v_fma_f32 v4, v5, v3, v4
+; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; VI-NEXT: v_div_fmas_f32 v6, v6, v10, v11
-; VI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v4
-; VI-NEXT: v_div_fixup_f32 v6, v6, v3, 1.0
-; VI-NEXT: s_cbranch_vccnz .LBB0_8
-; VI-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; VI-NEXT: v_sub_u32_e32 v4, vcc, v7, v8
-; VI-NEXT: v_add_u32_e32 v4, vcc, 11, v4
-; VI-NEXT: .LBB0_6: ; %frem.loop_body
+; VI-NEXT: s_cselect_b64 vcc, exec, 0
+; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_cmp_le_i32 s5, 11
+; VI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; VI-NEXT: s_cbranch_scc1 .LBB0_6
+; VI-NEXT: ; %bb.4: ; %frem.loop_body.preheader
+; VI-NEXT: s_add_i32 s7, s7, 11
+; VI-NEXT: s_sub_i32 s5, s7, s8
+; VI-NEXT: .LBB0_5: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v7, v5
-; VI-NEXT: v_mul_f32_e32 v5, v7, v6
-; VI-NEXT: v_rndne_f32_e32 v5, v5
-; VI-NEXT: v_fma_f32 v5, -v5, v3, v7
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v5
-; VI-NEXT: v_add_f32_e32 v8, v5, v3
-; VI-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc
-; VI-NEXT: v_add_u32_e32 v4, vcc, -11, v4
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v4
-; VI-NEXT: v_ldexp_f32 v5, v5, 11
-; VI-NEXT: s_cbranch_vccnz .LBB0_6
-; VI-NEXT: ; %bb.7: ; %Flow
-; VI-NEXT: v_mov_b32_e32 v5, v7
-; VI-NEXT: .LBB0_8: ; %frem.loop_exit
-; VI-NEXT: v_add_u32_e32 v4, vcc, -10, v4
-; VI-NEXT: v_ldexp_f32 v4, v5, v4
-; VI-NEXT: v_mul_f32_e32 v5, v4, v6
-; VI-NEXT: v_rndne_f32_e32 v5, v5
-; VI-NEXT: v_fma_f32 v4, -v5, v3, v4
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v4
-; VI-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; VI-NEXT: v_ldexp_f32 v2, v3, v2
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: s_movk_i32 s2, 0x7fff
-; VI-NEXT: v_bfi_b32 v2, s2, v2, v0
-; VI-NEXT: .LBB0_9: ; %Flow19
-; VI-NEXT: v_mov_b32_e32 v3, s0
-; VI-NEXT: s_movk_i32 s0, 0x7c00
-; VI-NEXT: v_mov_b32_e32 v4, s1
-; VI-NEXT: v_cmp_lg_f16_e32 vcc, 0, v1
-; VI-NEXT: v_cmp_nge_f16_e64 s[0:1], |v0|, s0
-; VI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; VI-NEXT: v_mov_b32_e32 v0, 0x7e00
-; VI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; VI-NEXT: flat_store_short v[3:4], v0
+; VI-NEXT: v_mul_f32_e32 v2, s6, v1
+; VI-NEXT: v_rndne_f32_e32 v2, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s6
+; VI-NEXT: v_readfirstlane_b32 s7, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; VI-NEXT: v_add_f32_e32 v2, v2, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s8, v2
+; VI-NEXT: s_cselect_b32 s7, s8, s7
+; VI-NEXT: v_ldexp_f32 v2, s7, 11
+; VI-NEXT: v_readfirstlane_b32 s8, v2
+; VI-NEXT: s_add_i32 s5, s5, -11
+; VI-NEXT: s_cmp_gt_i32 s5, 11
+; VI-NEXT: s_mov_b32 s7, s6
+; VI-NEXT: s_mov_b32 s6, s8
+; VI-NEXT: s_cbranch_scc1 .LBB0_5
+; VI-NEXT: s_branch .LBB0_7
+; VI-NEXT: .LBB0_6:
+; VI-NEXT: s_mov_b32 s7, s6
+; VI-NEXT: .LBB0_7: ; %frem.loop_exit
+; VI-NEXT: s_add_i32 s5, s5, -10
+; VI-NEXT: v_mov_b32_e32 v2, s5
+; VI-NEXT: v_ldexp_f32 v2, s7, v2
+; VI-NEXT: v_mul_f32_e32 v1, v2, v1
+; VI-NEXT: v_rndne_f32_e32 v1, v1
+; VI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: v_readfirstlane_b32 s5, v1
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s6, v0
+; VI-NEXT: s_cselect_b32 s5, s6, s5
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_ldexp_f32 v0, s5, v0
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: s_and_b32 s5, s2, 0xffff8000
+; VI-NEXT: v_readfirstlane_b32 s4, v0
+; VI-NEXT: s_and_b32 s4, s4, 0x7fff
+; VI-NEXT: s_or_b32 s4, s4, s5
+; VI-NEXT: .LBB0_8: ; %Flow19
+; VI-NEXT: v_cmp_nlg_f16_e64 s[6:7], s3, 0
+; VI-NEXT: v_mov_b32_e32 v0, 0x7c00
+; VI-NEXT: s_cmp_lg_u64 s[6:7], 0
+; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |s2|, v0
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: s_cselect_b32 s2, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 0
+; VI-NEXT: s_cselect_b32 s3, 0x7e00, s4
+; VI-NEXT: s_cmp_lg_u32 s2, 0
+; VI-NEXT: s_cselect_b32 s2, s3, 0x7e00
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
%gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
%r0 = load half, ptr addrspace(1) %in1, align 4
@@ -430,20 +462,21 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: s_sub_i32 s5, s7, s8
; CI-NEXT: .LBB3_5: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b32 s7, s6
-; CI-NEXT: v_mul_f32_e32 v2, s7, v1
+; CI-NEXT: v_mul_f32_e32 v2, s6, v1
; CI-NEXT: v_rndne_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v2, -v2, v0, s7
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s6
; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
-; CI-NEXT: v_readfirstlane_b32 s6, v2
+; CI-NEXT: v_readfirstlane_b32 s7, v2
; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
; CI-NEXT: v_add_f32_e32 v2, v2, v0
; CI-NEXT: v_readfirstlane_b32 s8, v2
-; CI-NEXT: s_cselect_b32 s6, s8, s6
-; CI-NEXT: v_ldexp_f32_e64 v2, s6, 12
+; CI-NEXT: s_cselect_b32 s7, s8, s7
+; CI-NEXT: v_ldexp_f32_e64 v2, s7, 12
+; CI-NEXT: v_readfirstlane_b32 s8, v2
; CI-NEXT: s_add_i32 s5, s5, -12
-; CI-NEXT: v_readfirstlane_b32 s6, v2
; CI-NEXT: s_cmp_gt_i32 s5, 12
+; CI-NEXT: s_mov_b32 s7, s6
+; CI-NEXT: s_mov_b32 s6, s8
; CI-NEXT: s_cbranch_scc1 .LBB3_5
; CI-NEXT: s_branch .LBB3_7
; CI-NEXT: .LBB3_6:
@@ -541,20 +574,21 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: s_sub_i32 s5, s7, s8
; VI-NEXT: .LBB3_5: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b32 s7, s6
-; VI-NEXT: v_mul_f32_e32 v2, s7, v1
+; VI-NEXT: v_mul_f32_e32 v2, s6, v1
; VI-NEXT: v_rndne_f32_e32 v2, v2
-; VI-NEXT: v_fma_f32 v2, -v2, v0, s7
-; VI-NEXT: v_readfirstlane_b32 s6, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s6
+; VI-NEXT: v_readfirstlane_b32 s7, v2
; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
; VI-NEXT: v_add_f32_e32 v2, v2, v0
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s8, v2
-; VI-NEXT: s_cselect_b32 s6, s8, s6
-; VI-NEXT: v_ldexp_f32 v2, s6, 12
+; VI-NEXT: s_cselect_b32 s7, s8, s7
+; VI-NEXT: v_ldexp_f32 v2, s7, 12
+; VI-NEXT: v_readfirstlane_b32 s8, v2
; VI-NEXT: s_add_i32 s5, s5, -12
-; VI-NEXT: v_readfirstlane_b32 s6, v2
; VI-NEXT: s_cmp_gt_i32 s5, 12
+; VI-NEXT: s_mov_b32 s7, s6
+; VI-NEXT: s_mov_b32 s6, s8
; VI-NEXT: s_cbranch_scc1 .LBB3_5
; VI-NEXT: s_branch .LBB3_7
; VI-NEXT: .LBB3_6:
@@ -805,23 +839,24 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; CI-NEXT: s_sub_i32 s11, s8, s9
; CI-NEXT: .LBB6_5: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b64 s[8:9], s[6:7]
-; CI-NEXT: v_mul_f64 v[4:5], s[8:9], v[2:3]
+; CI-NEXT: v_mul_f64 v[4:5], s[6:7], v[2:3]
; CI-NEXT: v_rndne_f64_e32 v[4:5], v[4:5]
-; CI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[8:9]
+; CI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[6:7]
; CI-NEXT: v_cmp_gt_f64_e32 vcc, 0, v[4:5]
-; CI-NEXT: v_readfirstlane_b32 s6, v4
-; CI-NEXT: v_readfirstlane_b32 s7, v5
+; CI-NEXT: v_readfirstlane_b32 s8, v4
+; CI-NEXT: v_readfirstlane_b32 s9, v5
; CI-NEXT: v_add_f64 v[4:5], v[4:5], v[0:1]
; CI-NEXT: s_or_b64 s[12:13], vcc, vcc
; CI-NEXT: v_readfirstlane_b32 s12, v4
; CI-NEXT: v_readfirstlane_b32 s13, v5
-; CI-NEXT: s_cselect_b64 s[6:7], s[12:13], s[6:7]
-; CI-NEXT: v_ldexp_f64 v[4:5], s[6:7], 26
+; CI-NEXT: s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; CI-NEXT: v_ldexp_f64 v[4:5], s[8:9], 26
; CI-NEXT: s_sub_i32 s11, s11, 26
; CI-NEXT: s_cmp_gt_i32 s11, 26
-; CI-NEXT: v_readfirstlane_b32 s6, v4
-; CI-NEXT: v_readfirstlane_b32 s7, v5
+; CI-NEXT: s_mov_b64 s[8:9], s[6:7]
+; CI-NEXT: v_readfirstlane_b32 s12, v4
+; CI-NEXT: v_readfirstlane_b32 s13, v5
+; CI-NEXT: s_mov_b64 s[6:7], s[12:13]
; CI-NEXT: s_cbranch_scc1 .LBB6_5
; CI-NEXT: s_branch .LBB6_7
; CI-NEXT: .LBB6_6:
@@ -933,23 +968,24 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; VI-NEXT: s_sub_i32 s11, s8, s9
; VI-NEXT: .LBB6_5: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b64 s[8:9], s[6:7]
-; VI-NEXT: v_mul_f64 v[4:5], s[8:9], v[2:3]
+; VI-NEXT: v_mul_f64 v[4:5], s[6:7], v[2:3]
; VI-NEXT: v_rndne_f64_e32 v[4:5], v[4:5]
-; VI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[8:9]
-; VI-NEXT: v_readfirstlane_b32 s6, v4
-; VI-NEXT: v_readfirstlane_b32 s7, v5
+; VI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[6:7]
+; VI-NEXT: v_readfirstlane_b32 s8, v4
+; VI-NEXT: v_readfirstlane_b32 s9, v5
; VI-NEXT: v_cmp_gt_f64_e32 vcc, 0, v[4:5]
; VI-NEXT: v_add_f64 v[4:5], v[4:5], v[0:1]
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s12, v4
; VI-NEXT: v_readfirstlane_b32 s13, v5
-; VI-NEXT: s_cselect_b64 s[6:7], s[12:13], s[6:7]
-; VI-NEXT: v_ldexp_f64 v[4:5], s[6:7], 26
+; VI-NEXT: s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; VI-NEXT: v_ldexp_f64 v[4:5], s[8:9], 26
; VI-NEXT: s_sub_i32 s11, s11, 26
; VI-NEXT: s_cmp_gt_i32 s11, 26
-; VI-NEXT: v_readfirstlane_b32 s6, v4
-; VI-NEXT: v_readfirstlane_b32 s7, v5
+; VI-NEXT: s_mov_b64 s[8:9], s[6:7]
+; VI-NEXT: v_readfirstlane_b32 s12, v4
+; VI-NEXT: v_readfirstlane_b32 s13, v5
+; VI-NEXT: s_mov_b64 s[6:7], s[12:13]
; VI-NEXT: s_cbranch_scc1 .LBB6_5
; VI-NEXT: s_branch .LBB6_7
; VI-NEXT: .LBB6_6:
@@ -1140,375 +1176,459 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %in1, ptr addrspace(1) %in2) #0 {
; CI-LABEL: frem_v2f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; CI-NEXT: s_mov_b32 s7, 0xf000
-; CI-NEXT: s_mov_b32 s6, -1
-; CI-NEXT: s_mov_b32 s2, s6
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_load_dword s2, s[2:3], 0x0
+; CI-NEXT: s_load_dword s3, s[4:5], 0x4
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s4, s10
-; CI-NEXT: s_mov_b32 s5, s11
-; CI-NEXT: s_mov_b32 s3, s7
-; CI-NEXT: buffer_load_dword v0, off, s[4:7], 0
-; CI-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:16
-; CI-NEXT: s_waitcnt vmcnt(1)
-; CI-NEXT: v_cvt_f32_f16_e64 v4, |v0|
-; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e64 v3, |v1|
-; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v4, v3
-; CI-NEXT: s_cbranch_vccz .LBB9_2
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s3|
+; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[4:5], vcc, vcc
+; CI-NEXT: s_mov_b32 s5, 1
+; CI-NEXT: ; implicit-def: $sgpr4
+; CI-NEXT: s_cbranch_scc0 .LBB9_2
; CI-NEXT: ; %bb.1: ; %frem.else20
-; CI-NEXT: v_and_b32_e32 v2, 0xffff8000, v0
-; CI-NEXT: v_cmp_eq_f32_e32 vcc, v4, v3
-; CI-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
-; CI-NEXT: s_mov_b64 s[0:1], 0
-; CI-NEXT: s_branch .LBB9_3
-; CI-NEXT: .LBB9_2:
-; CI-NEXT: s_mov_b64 s[0:1], -1
-; CI-NEXT: ; implicit-def: $vgpr2
-; CI-NEXT: .LBB9_3: ; %Flow57
-; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: s_cselect_b32 s0, 1, 0
-; CI-NEXT: s_cmp_lg_u32 s0, 1
-; CI-NEXT: s_cbranch_scc1 .LBB9_9
-; CI-NEXT: ; %bb.4: ; %frem.compute19
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v7, v4
-; CI-NEXT: v_frexp_mant_f32_e32 v2, v4
-; CI-NEXT: v_frexp_mant_f32_e32 v4, v3
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v8, v3
-; CI-NEXT: v_ldexp_f32_e64 v3, v4, 1
-; CI-NEXT: v_div_scale_f32 v9, s[0:1], v3, v3, 1.0
-; CI-NEXT: v_ldexp_f32_e64 v5, v2, 11
-; CI-NEXT: v_add_i32_e32 v2, vcc, -1, v8
-; CI-NEXT: v_not_b32_e32 v4, v2
-; CI-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; CI-NEXT: v_div_scale_f32 v6, vcc, 1.0, v3, 1.0
-; CI-NEXT: v_rcp_f32_e32 v10, v9
+; CI-NEXT: s_and_b32 s4, s2, 0xffff8000
+; CI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[6:7], vcc, vcc
+; CI-NEXT: s_mov_b32 s5, 0
+; CI-NEXT: s_cselect_b32 s4, s4, s2
+; CI-NEXT: .LBB9_2: ; %Flow57
+; CI-NEXT: s_xor_b32 s5, s5, 1
+; CI-NEXT: s_cmp_lg_u32 s5, 0
+; CI-NEXT: s_cbranch_scc1 .LBB9_8
+; CI-NEXT: ; %bb.3: ; %frem.compute19
+; CI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; CI-NEXT: v_readfirstlane_b32 s7, v1
+; CI-NEXT: v_ldexp_f32_e64 v1, v2, 11
+; CI-NEXT: v_readfirstlane_b32 s6, v1
+; CI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; CI-NEXT: v_readfirstlane_b32 s8, v0
+; CI-NEXT: v_ldexp_f32_e64 v0, v1, 1
+; CI-NEXT: v_div_scale_f32 v1, s[10:11], v0, v0, 1.0
+; CI-NEXT: s_add_i32 s5, s7, -1
+; CI-NEXT: s_add_i32 s4, s8, -1
+; CI-NEXT: s_sub_i32 s5, s5, s4
+; CI-NEXT: v_div_scale_f32 v2, s[10:11], 1.0, v0, 1.0
+; CI-NEXT: s_or_b64 s[10:11], s[10:11], s[10:11]
+; CI-NEXT: v_rcp_f32_e32 v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v11, -v9, v10, 1.0
-; CI-NEXT: v_fma_f32 v10, v11, v10, v10
-; CI-NEXT: v_mul_f32_e32 v11, v6, v10
-; CI-NEXT: v_fma_f32 v12, -v9, v11, v6
-; CI-NEXT: v_fma_f32 v11, v12, v10, v11
-; CI-NEXT: v_fma_f32 v6, -v9, v11, v6
+; CI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; CI-NEXT: v_fma_f32 v3, v4, v3, v3
+; CI-NEXT: v_mul_f32_e32 v4, v2, v3
+; CI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; CI-NEXT: v_fma_f32 v4, v5, v3, v4
+; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v6, v6, v10, v11
-; CI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v4
-; CI-NEXT: v_div_fixup_f32 v6, v6, v3, 1.0
-; CI-NEXT: s_cbranch_vccnz .LBB9_8
-; CI-NEXT: ; %bb.5: ; %frem.loop_body27.preheader
-; CI-NEXT: v_sub_i32_e32 v4, vcc, v7, v8
-; CI-NEXT: v_add_i32_e32 v4, vcc, 11, v4
-; CI-NEXT: .LBB9_6: ; %frem.loop_body27
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_cmp_le_i32 s5, 11
+; CI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; CI-NEXT: s_cbranch_scc1 .LBB9_6
+; CI-NEXT: ; %bb.4: ; %frem.loop_body27.preheader
+; CI-NEXT: s_add_i32 s7, s7, 11
+; CI-NEXT: s_sub_i32 s5, s7, s8
+; CI-NEXT: .LBB9_5: ; %frem.loop_body27
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: v_mov_b32_e32 v7, v5
-; CI-NEXT: v_mul_f32_e32 v5, v7, v6
-; CI-NEXT: v_rndne_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v5, -v5, v3, v7
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v5
-; CI-NEXT: v_add_f32_e32 v8, v5, v3
-; CI-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc
-; CI-NEXT: v_add_i32_e32 v4, vcc, -11, v4
-; CI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v4
-; CI-NEXT: v_ldexp_f32_e64 v5, v5, 11
-; CI-NEXT: s_cbranch_vccnz .LBB9_6
-; CI-NEXT: ; %bb.7: ; %Flow55
-; CI-NEXT: v_mov_b32_e32 v5, v7
-; CI-NEXT: .LBB9_8: ; %frem.loop_exit28
-; CI-NEXT: v_add_i32_e32 v4, vcc, -10, v4
-; CI-NEXT: v_ldexp_f32_e32 v4, v5, v4
-; CI-NEXT: v_mul_f32_e32 v5, v4, v6
-; CI-NEXT: v_rndne_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v4, -v5, v3, v4
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v4
-; CI-NEXT: v_add_f32_e32 v3, v4, v3
-; CI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; CI-NEXT: v_ldexp_f32_e32 v2, v3, v2
-; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; CI-NEXT: v_and_b32_e32 v3, 0xffff8000, v0
-; CI-NEXT: v_and_b32_e32 v2, 0x7fff, v2
-; CI-NEXT: v_or_b32_e32 v2, v2, v3
-; CI-NEXT: .LBB9_9: ; %Flow58
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; CI-NEXT: v_cvt_f32_f16_e64 v7, |v3|
-; CI-NEXT: v_cvt_f32_f16_e64 v6, |v4|
-; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v7, v6
-; CI-NEXT: s_cbranch_vccz .LBB9_11
-; CI-NEXT: ; %bb.10: ; %frem.else
-; CI-NEXT: v_and_b32_e32 v5, 0x8000, v3
-; CI-NEXT: v_cmp_eq_f32_e32 vcc, v7, v6
-; CI-NEXT: v_cndmask_b32_e32 v5, v3, v5, vcc
-; CI-NEXT: s_mov_b64 s[0:1], 0
-; CI-NEXT: s_branch .LBB9_12
-; CI-NEXT: .LBB9_11:
-; CI-NEXT: s_mov_b64 s[0:1], -1
-; CI-NEXT: ; implicit-def: $vgpr5
-; CI-NEXT: .LBB9_12: ; %Flow53
-; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: s_cselect_b32 s0, 1, 0
-; CI-NEXT: s_cmp_lg_u32 s0, 1
-; CI-NEXT: s_cbranch_scc1 .LBB9_18
-; CI-NEXT: ; %bb.13: ; %frem.compute
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v10, v7
-; CI-NEXT: v_frexp_mant_f32_e32 v5, v7
-; CI-NEXT: v_frexp_mant_f32_e32 v7, v6
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v11, v6
-; CI-NEXT: v_ldexp_f32_e64 v6, v7, 1
-; CI-NEXT: v_div_scale_f32 v12, s[0:1], v6, v6, 1.0
-; CI-NEXT: v_ldexp_f32_e64 v8, v5, 11
-; CI-NEXT: v_add_i32_e32 v5, vcc, -1, v11
-; CI-NEXT: v_not_b32_e32 v7, v5
-; CI-NEXT: v_add_i32_e32 v7, vcc, v7, v10
-; CI-NEXT: v_div_scale_f32 v9, vcc, 1.0, v6, 1.0
-; CI-NEXT: v_rcp_f32_e32 v13, v12
+; CI-NEXT: v_mul_f32_e32 v2, s6, v1
+; CI-NEXT: v_rndne_f32_e32 v2, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s6
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s7, v2
+; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v2, v2, v0
+; CI-NEXT: v_readfirstlane_b32 s8, v2
+; CI-NEXT: s_cselect_b32 s7, s8, s7
+; CI-NEXT: v_ldexp_f32_e64 v2, s7, 11
+; CI-NEXT: v_readfirstlane_b32 s8, v2
+; CI-NEXT: s_add_i32 s5, s5, -11
+; CI-NEXT: s_cmp_gt_i32 s5, 11
+; CI-NEXT: s_mov_b32 s7, s6
+; CI-NEXT: s_mov_b32 s6, s8
+; CI-NEXT: s_cbranch_scc1 .LBB9_5
+; CI-NEXT: s_branch .LBB9_7
+; CI-NEXT: .LBB9_6:
+; CI-NEXT: s_mov_b32 s7, s6
+; CI-NEXT: .LBB9_7: ; %frem.loop_exit28
+; CI-NEXT: s_add_i32 s5, s5, -10
+; CI-NEXT: v_mov_b32_e32 v2, s5
+; CI-NEXT: v_ldexp_f32_e32 v2, s7, v2
+; CI-NEXT: v_mul_f32_e32 v1, v2, v1
+; CI-NEXT: v_rndne_f32_e32 v1, v1
+; CI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; CI-NEXT: s_or_b64 s[6:7], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v0, v1, v0
+; CI-NEXT: v_readfirstlane_b32 s5, v1
+; CI-NEXT: v_readfirstlane_b32 s6, v0
+; CI-NEXT: s_cselect_b32 s5, s6, s5
+; CI-NEXT: v_mov_b32_e32 v0, s4
+; CI-NEXT: v_ldexp_f32_e32 v0, s5, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CI-NEXT: s_and_b32 s5, s2, 0xffff8000
+; CI-NEXT: v_readfirstlane_b32 s4, v0
+; CI-NEXT: s_and_b32 s4, s4, 0x7fff
+; CI-NEXT: s_or_b32 s4, s4, s5
+; CI-NEXT: .LBB9_8: ; %Flow58
+; CI-NEXT: s_lshr_b32 s5, s2, 16
+; CI-NEXT: s_lshr_b32 s6, s3, 16
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s5|
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s6|
+; CI-NEXT: ; implicit-def: $sgpr7
+; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
+; CI-NEXT: s_mov_b32 s8, 1
+; CI-NEXT: s_cbranch_scc0 .LBB9_10
+; CI-NEXT: ; %bb.9: ; %frem.else
+; CI-NEXT: s_and_b32 s7, s5, 0xffff8000
+; CI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
+; CI-NEXT: s_mov_b32 s8, 0
+; CI-NEXT: s_cselect_b32 s7, s7, s5
+; CI-NEXT: .LBB9_10: ; %Flow53
+; CI-NEXT: s_xor_b32 s8, s8, 1
+; CI-NEXT: s_cmp_lg_u32 s8, 0
+; CI-NEXT: s_cbranch_scc1 .LBB9_16
+; CI-NEXT: ; %bb.11: ; %frem.compute
+; CI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; CI-NEXT: v_readfirstlane_b32 s10, v1
+; CI-NEXT: v_ldexp_f32_e64 v1, v2, 11
+; CI-NEXT: v_readfirstlane_b32 s9, v1
+; CI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; CI-NEXT: v_readfirstlane_b32 s11, v0
+; CI-NEXT: v_ldexp_f32_e64 v0, v1, 1
+; CI-NEXT: v_div_scale_f32 v1, s[12:13], v0, v0, 1.0
+; CI-NEXT: s_add_i32 s8, s10, -1
+; CI-NEXT: s_add_i32 s7, s11, -1
+; CI-NEXT: s_sub_i32 s8, s8, s7
+; CI-NEXT: v_div_scale_f32 v2, s[12:13], 1.0, v0, 1.0
+; CI-NEXT: s_or_b64 s[12:13], s[12:13], s[12:13]
+; CI-NEXT: v_rcp_f32_e32 v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v14, -v12, v13, 1.0
-; CI-NEXT: v_fma_f32 v13, v14, v13, v13
-; CI-NEXT: v_mul_f32_e32 v14, v9, v13
-; CI-NEXT: v_fma_f32 v15, -v12, v14, v9
-; CI-NEXT: v_fma_f32 v14, v15, v13, v14
-; CI-NEXT: v_fma_f32 v9, -v12, v14, v9
+; CI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; CI-NEXT: v_fma_f32 v3, v4, v3, v3
+; CI-NEXT: v_mul_f32_e32 v4, v2, v3
+; CI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; CI-NEXT: v_fma_f32 v4, v5, v3, v4
+; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v9, v9, v13, v14
-; CI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v7
-; CI-NEXT: v_div_fixup_f32 v9, v9, v6, 1.0
-; CI-NEXT: s_cbranch_vccnz .LBB9_17
-; CI-NEXT: ; %bb.14: ; %frem.loop_body.preheader
-; CI-NEXT: v_sub_i32_e32 v7, vcc, v10, v11
-; CI-NEXT: v_add_i32_e32 v7, vcc, 11, v7
-; CI-NEXT: .LBB9_15: ; %frem.loop_body
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_cmp_le_i32 s8, 11
+; CI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; CI-NEXT: s_cbranch_scc1 .LBB9_14
+; CI-NEXT: ; %bb.12: ; %frem.loop_body.preheader
+; CI-NEXT: s_add_i32 s10, s10, 11
+; CI-NEXT: s_sub_i32 s8, s10, s11
+; CI-NEXT: .LBB9_13: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: v_mov_b32_e32 v10, v8
-; CI-NEXT: v_mul_f32_e32 v8, v10, v9
-; CI-NEXT: v_rndne_f32_e32 v8, v8
-; CI-NEXT: v_fma_f32 v8, -v8, v6, v10
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v8
-; CI-NEXT: v_add_f32_e32 v11, v8, v6
-; CI-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
-; CI-NEXT: v_add_i32_e32 v7, vcc, -11, v7
-; CI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v7
-; CI-NEXT: v_ldexp_f32_e64 v8, v8, 11
-; CI-NEXT: s_cbranch_vccnz .LBB9_15
-; CI-NEXT: ; %bb.16: ; %Flow
-; CI-NEXT: v_mov_b32_e32 v8, v10
-; CI-NEXT: .LBB9_17: ; %frem.loop_exit
-; CI-NEXT: v_add_i32_e32 v7, vcc, -10, v7
-; CI-NEXT: v_ldexp_f32_e32 v7, v8, v7
-; CI-NEXT: v_mul_f32_e32 v8, v7, v9
-; CI-NEXT: v_rndne_f32_e32 v8, v8
-; CI-NEXT: v_fma_f32 v7, -v8, v6, v7
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v7
-; CI-NEXT: v_add_f32_e32 v6, v7, v6
-; CI-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
-; CI-NEXT: v_ldexp_f32_e32 v5, v6, v5
-; CI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; CI-NEXT: v_and_b32_e32 v6, 0x8000, v3
-; CI-NEXT: v_and_b32_e32 v5, 0x7fff, v5
-; CI-NEXT: v_or_b32_e32 v5, v5, v6
-; CI-NEXT: .LBB9_18: ; %Flow54
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; CI-NEXT: s_mov_b32 s2, 0x7f800000
-; CI-NEXT: s_mov_b32 s11, 0xf000
-; CI-NEXT: v_cmp_lg_f32_e32 vcc, 0, v1
-; CI-NEXT: v_cmp_nle_f32_e64 s[0:1], s2, v0
-; CI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; CI-NEXT: v_mov_b32_e32 v0, 0x7e00
-; CI-NEXT: v_cndmask_b32_e32 v1, v0, v2, vcc
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v4
-; CI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; CI-NEXT: s_mov_b32 s10, -1
-; CI-NEXT: v_cmp_lg_f32_e32 vcc, 0, v2
-; CI-NEXT: v_cvt_f32_f16_e64 v2, |v3|
-; CI-NEXT: v_cmp_nle_f32_e64 s[0:1], s2, v2
-; CI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; CI-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_or_b32_e32 v0, v1, v0
-; CI-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: frem_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: s_add_u32 s2, s4, 16
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: s_addc_u32 s3, s5, 0
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_load_dword v1, v[1:2]
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_f16_e64 v4, |v0|
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e64 v3, |v1|
-; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v4, v3
-; VI-NEXT: s_cbranch_vccz .LBB9_2
-; VI-NEXT: ; %bb.1: ; %frem.else20
-; VI-NEXT: v_and_b32_e32 v2, 0x8000, v0
-; VI-NEXT: v_cmp_eq_f32_e32 vcc, v4, v3
-; VI-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
-; VI-NEXT: s_mov_b64 s[2:3], 0
-; VI-NEXT: s_branch .LBB9_3
-; VI-NEXT: .LBB9_2:
-; VI-NEXT: s_mov_b64 s[2:3], -1
-; VI-NEXT: ; implicit-def: $vgpr2
-; VI-NEXT: .LBB9_3: ; %Flow57
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cbranch_scc1 .LBB9_9
-; VI-NEXT: ; %bb.4: ; %frem.compute19
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v7, v4
-; VI-NEXT: v_frexp_mant_f32_e32 v2, v4
-; VI-NEXT: v_frexp_mant_f32_e32 v4, v3
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v8, v3
-; VI-NEXT: v_ldexp_f32 v3, v4, 1
-; VI-NEXT: v_div_scale_f32 v9, s[2:3], v3, v3, 1.0
-; VI-NEXT: v_ldexp_f32 v5, v2, 11
-; VI-NEXT: v_add_u32_e32 v2, vcc, -1, v8
-; VI-NEXT: v_not_b32_e32 v4, v2
-; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v7
-; VI-NEXT: v_div_scale_f32 v6, vcc, 1.0, v3, 1.0
-; VI-NEXT: v_rcp_f32_e32 v10, v9
+; CI-NEXT: v_mul_f32_e32 v2, s9, v1
+; CI-NEXT: v_rndne_f32_e32 v2, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s9
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s12, v2
+; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v2, v2, v0
+; CI-NEXT: v_readfirstlane_b32 s10, v2
+; CI-NEXT: s_cselect_b32 s10, s10, s12
+; CI-NEXT: v_ldexp_f32_e64 v2, s10, 11
+; CI-NEXT: v_readfirstlane_b32 s11, v2
+; CI-NEXT: s_add_i32 s8, s8, -11
+; CI-NEXT: s_cmp_gt_i32 s8, 11
+; CI-NEXT: s_mov_b32 s10, s9
+; CI-NEXT: s_mov_b32 s9, s11
+; CI-NEXT: s_cbranch_scc1 .LBB9_13
+; CI-NEXT: s_branch .LBB9_15
+; CI-NEXT: .LBB9_14:
+; CI-NEXT: s_mov_b32 s10, s9
+; CI-NEXT: .LBB9_15: ; %frem.loop_exit
+; CI-NEXT: s_add_i32 s8, s8, -10
+; CI-NEXT: v_mov_b32_e32 v2, s8
+; CI-NEXT: v_ldexp_f32_e32 v2, s10, v2
+; CI-NEXT: v_mul_f32_e32 v1, v2, v1
+; CI-NEXT: v_rndne_f32_e32 v1, v1
+; CI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v0, v1, v0
+; CI-NEXT: v_readfirstlane_b32 s10, v1
+; CI-NEXT: v_readfirstlane_b32 s8, v0
+; CI-NEXT: s_cselect_b32 s8, s8, s10
+; CI-NEXT: v_mov_b32_e32 v0, s7
+; CI-NEXT: v_ldexp_f32_e32 v0, s8, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CI-NEXT: s_and_b32 s8, s5, 0xffff8000
+; CI-NEXT: v_readfirstlane_b32 s7, v0
+; CI-NEXT: s_and_b32 s7, s7, 0x7fff
+; CI-NEXT: s_or_b32 s7, s7, s8
+; CI-NEXT: .LBB9_16: ; %Flow54
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; CI-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s2|
+; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
+; CI-NEXT: s_cselect_b32 s8, 1, 0
+; CI-NEXT: s_movk_i32 s9, 0x7e00
+; CI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s6
+; CI-NEXT: s_or_b64 s[2:3], vcc, vcc
+; CI-NEXT: s_cselect_b32 s10, 1, 0
+; CI-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s5|
+; CI-NEXT: s_or_b64 s[2:3], vcc, vcc
+; CI-NEXT: s_cselect_b32 s6, 1, 0
+; CI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
+; CI-NEXT: s_or_b64 s[2:3], vcc, vcc
+; CI-NEXT: s_cselect_b32 s2, 1, 0
+; CI-NEXT: s_bfe_u32 s3, s9, 0x100000
+; CI-NEXT: s_bfe_u32 s4, s4, 0x100000
+; CI-NEXT: s_cmp_lg_u32 s8, 0
+; CI-NEXT: s_cselect_b32 s4, s3, s4
+; CI-NEXT: s_cmp_lg_u32 s10, 0
+; CI-NEXT: s_cselect_b32 s4, s4, s3
+; CI-NEXT: s_bfe_u32 s5, s7, 0x100000
+; CI-NEXT: s_cmp_lg_u32 s6, 0
+; CI-NEXT: s_cselect_b32 s5, s3, s5
+; CI-NEXT: s_cmp_lg_u32 s2, 0
+; CI-NEXT: s_cselect_b32 s2, s5, s3
+; CI-NEXT: s_lshl_b32 s2, s2, 16
+; CI-NEXT: s_or_b32 s2, s4, s2
+; CI-NEXT: v_mov_b32_e32 v0, s2
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; VI-LABEL: frem_v2f16:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-NEXT: s_load_dword s3, s[4:5], 0x10
+; VI-NEXT: s_mov_b32 s5, 1
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
+; VI-NEXT: v_cvt_f32_f16_e64 v0, |s3|
+; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
+; VI-NEXT: ; %bb.1: ; %frem.else20
+; VI-NEXT: s_and_b32 s4, s2, 0xffff8000
+; VI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_cselect_b32 s4, s4, s2
+; VI-NEXT: .LBB9_2: ; %Flow57
+; VI-NEXT: s_xor_b32 s5, s5, 1
+; VI-NEXT: s_cmp_lg_u32 s5, 0
+; VI-NEXT: s_cbranch_scc1 .LBB9_8
+; VI-NEXT: ; %bb.3: ; %frem.compute19
+; VI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; VI-NEXT: v_readfirstlane_b32 s7, v1
+; VI-NEXT: v_ldexp_f32 v1, v2, 11
+; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; VI-NEXT: v_readfirstlane_b32 s8, v0
+; VI-NEXT: v_ldexp_f32 v0, v1, 1
+; VI-NEXT: v_div_scale_f32 v1, s[10:11], v0, v0, 1.0
+; VI-NEXT: s_add_i32 s5, s7, -1
+; VI-NEXT: s_add_i32 s4, s8, -1
+; VI-NEXT: s_sub_i32 s5, s5, s4
+; VI-NEXT: v_div_scale_f32 v2, s[10:11], 1.0, v0, 1.0
+; VI-NEXT: s_cmp_lg_u64 s[10:11], 0
+; VI-NEXT: v_rcp_f32_e32 v3, v1
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; VI-NEXT: v_fma_f32 v11, -v9, v10, 1.0
-; VI-NEXT: v_fma_f32 v10, v11, v10, v10
-; VI-NEXT: v_mul_f32_e32 v11, v6, v10
-; VI-NEXT: v_fma_f32 v12, -v9, v11, v6
-; VI-NEXT: v_fma_f32 v11, v12, v10, v11
-; VI-NEXT: v_fma_f32 v6, -v9, v11, v6
+; VI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; VI-NEXT: v_fma_f32 v3, v4, v3, v3
+; VI-NEXT: v_mul_f32_e32 v4, v2, v3
+; VI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; VI-NEXT: v_fma_f32 v4, v5, v3, v4
+; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; VI-NEXT: v_div_fmas_f32 v6, v6, v10, v11
-; VI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v4
-; VI-NEXT: v_div_fixup_f32 v6, v6, v3, 1.0
-; VI-NEXT: s_cbranch_vccnz .LBB9_8
-; VI-NEXT: ; %bb.5: ; %frem.loop_body27.preheader
-; VI-NEXT: v_sub_u32_e32 v4, vcc, v7, v8
-; VI-NEXT: v_add_u32_e32 v4, vcc, 11, v4
-; VI-NEXT: .LBB9_6: ; %frem.loop_body27
+; VI-NEXT: s_cselect_b64 vcc, exec, 0
+; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_cmp_le_i32 s5, 11
+; VI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; VI-NEXT: s_cbranch_scc1 .LBB9_6
+; VI-NEXT: ; %bb.4: ; %frem.loop_body27.preheader
+; VI-NEXT: s_add_i32 s7, s7, 11
+; VI-NEXT: s_sub_i32 s5, s7, s8
+; VI-NEXT: .LBB9_5: ; %frem.loop_body27
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v7, v5
-; VI-NEXT: v_mul_f32_e32 v5, v7, v6
-; VI-NEXT: v_rndne_f32_e32 v5, v5
-; VI-NEXT: v_fma_f32 v5, -v5, v3, v7
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v5
-; VI-NEXT: v_add_f32_e32 v8, v5, v3
-; VI-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc
-; VI-NEXT: v_add_u32_e32 v4, vcc, -11, v4
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v4
-; VI-NEXT: v_ldexp_f32 v5, v5, 11
-; VI-NEXT: s_cbranch_vccnz .LBB9_6
-; VI-NEXT: ; %bb.7: ; %Flow55
-; VI-NEXT: v_mov_b32_e32 v5, v7
-; VI-NEXT: .LBB9_8: ; %frem.loop_exit28
-; VI-NEXT: v_add_u32_e32 v4, vcc, -10, v4
-; VI-NEXT: v_ldexp_f32 v4, v5, v4
-; VI-NEXT: v_mul_f32_e32 v5, v4, v6
-; VI-NEXT: v_rndne_f32_e32 v5, v5
-; VI-NEXT: v_fma_f32 v4, -v5, v3, v4
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v4
-; VI-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; VI-NEXT: v_ldexp_f32 v2, v3, v2
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: s_movk_i32 s2, 0x7fff
-; VI-NEXT: v_bfi_b32 v2, s2, v2, v0
-; VI-NEXT: .LBB9_9:
-; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; VI-NEXT: v_cvt_f32_f16_e64 v7, |v3|
-; VI-NEXT: v_cvt_f32_f16_e64 v6, |v4|
-; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v7, v6
-; VI-NEXT: s_cbranch_vccz .LBB9_11
-; VI-NEXT: ; %bb.10: ; %frem.else
-; VI-NEXT: v_and_b32_e32 v5, 0x8000, v3
-; VI-NEXT: v_cmp_eq_f32_e32 vcc, v7, v6
-; VI-NEXT: v_cndmask_b32_e32 v5, v3, v5, vcc
-; VI-NEXT: s_mov_b64 s[2:3], 0
-; VI-NEXT: s_branch .LBB9_12
-; VI-NEXT: .LBB9_11:
-; VI-NEXT: s_mov_b64 s[2:3], -1
-; VI-NEXT: ; implicit-def: $vgpr5
-; VI-NEXT: .LBB9_12: ; %Flow53
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cbranch_scc1 .LBB9_18
-; VI-NEXT: ; %bb.13: ; %frem.compute
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v10, v7
-; VI-NEXT: v_frexp_mant_f32_e32 v5, v7
-; VI-NEXT: v_frexp_mant_f32_e32 v7, v6
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v11, v6
-; VI-NEXT: v_ldexp_f32 v6, v7, 1
-; VI-NEXT: v_div_scale_f32 v12, s[2:3], v6, v6, 1.0
-; VI-NEXT: v_ldexp_f32 v8, v5, 11
-; VI-NEXT: v_add_u32_e32 v5, vcc, -1, v11
-; VI-NEXT: v_not_b32_e32 v7, v5
-; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v10
-; VI-NEXT: v_div_scale_f32 v9, vcc, 1.0, v6, 1.0
-; VI-NEXT: v_rcp_f32_e32 v13, v12
+; VI-NEXT: v_mul_f32_e32 v2, s6, v1
+; VI-NEXT: v_rndne_f32_e32 v2, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s6
+; VI-NEXT: v_readfirstlane_b32 s7, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; VI-NEXT: v_add_f32_e32 v2, v2, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s8, v2
+; VI-NEXT: s_cselect_b32 s7, s8, s7
+; VI-NEXT: v_ldexp_f32 v2, s7, 11
+; VI-NEXT: v_readfirstlane_b32 s8, v2
+; VI-NEXT: s_add_i32 s5, s5, -11
+; VI-NEXT: s_cmp_gt_i32 s5, 11
+; VI-NEXT: s_mov_b32 s7, s6
+; VI-NEXT: s_mov_b32 s6, s8
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: s_branch .LBB9_7
+; VI-NEXT: .LBB9_6:
+; VI-NEXT: s_mov_b32 s7, s6
+; VI-NEXT: .LBB9_7: ; %frem.loop_exit28
+; VI-NEXT: s_add_i32 s5, s5, -10
+; VI-NEXT: v_mov_b32_e32 v2, s5
+; VI-NEXT: v_ldexp_f32 v2, s7, v2
+; VI-NEXT: v_mul_f32_e32 v1, v2, v1
+; VI-NEXT: v_rndne_f32_e32 v1, v1
+; VI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: v_readfirstlane_b32 s5, v1
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s6, v0
+; VI-NEXT: s_cselect_b32 s5, s6, s5
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_ldexp_f32 v0, s5, v0
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: s_and_b32 s5, s2, 0xffff8000
+; VI-NEXT: v_readfirstlane_b32 s4, v0
+; VI-NEXT: s_and_b32 s4, s4, 0x7fff
+; VI-NEXT: s_or_b32 s4, s4, s5
+; VI-NEXT: .LBB9_8: ; %Flow58
+; VI-NEXT: s_lshr_b32 s5, s2, 16
+; VI-NEXT: s_lshr_b32 s6, s3, 16
+; VI-NEXT: v_cvt_f32_f16_e64 v1, |s5|
+; VI-NEXT: v_cvt_f32_f16_e64 v0, |s6|
+; VI-NEXT: s_mov_b32 s8, 1
+; VI-NEXT: ; implicit-def: $sgpr7
+; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_cbranch_scc0 .LBB9_10
+; VI-NEXT: ; %bb.9: ; %frem.else
+; VI-NEXT: s_and_b32 s7, s5, 0xffff8000
+; VI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_mov_b32 s8, 0
+; VI-NEXT: s_cselect_b32 s7, s7, s5
+; VI-NEXT: .LBB9_10: ; %Flow53
+; VI-NEXT: s_xor_b32 s8, s8, 1
+; VI-NEXT: s_cmp_lg_u32 s8, 0
+; VI-NEXT: s_cbranch_scc1 .LBB9_16
+; VI-NEXT: ; %bb.11: ; %frem.compute
+; VI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; VI-NEXT: v_readfirstlane_b32 s10, v1
+; VI-NEXT: v_ldexp_f32 v1, v2, 11
+; VI-NEXT: v_readfirstlane_b32 s9, v1
+; VI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; VI-NEXT: v_readfirstlane_b32 s11, v0
+; VI-NEXT: v_ldexp_f32 v0, v1, 1
+; VI-NEXT: v_div_scale_f32 v1, s[12:13], v0, v0, 1.0
+; VI-NEXT: s_add_i32 s8, s10, -1
+; VI-NEXT: s_add_i32 s7, s11, -1
+; VI-NEXT: s_sub_i32 s8, s8, s7
+; VI-NEXT: v_div_scale_f32 v2, s[12:13], 1.0, v0, 1.0
+; VI-NEXT: s_cmp_lg_u64 s[12:13], 0
+; VI-NEXT: v_rcp_f32_e32 v3, v1
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; VI-NEXT: v_fma_f32 v14, -v12, v13, 1.0
-; VI-NEXT: v_fma_f32 v13, v14, v13, v13
-; VI-NEXT: v_mul_f32_e32 v14, v9, v13
-; VI-NEXT: v_fma_f32 v15, -v12, v14, v9
-; VI-NEXT: v_fma_f32 v14, v15, v13, v14
-; VI-NEXT: v_fma_f32 v9, -v12, v14, v9
+; VI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; VI-NEXT: v_fma_f32 v3, v4, v3, v3
+; VI-NEXT: v_mul_f32_e32 v4, v2, v3
+; VI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; VI-NEXT: v_fma_f32 v4, v5, v3, v4
+; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; VI-NEXT: v_div_fmas_f32 v9, v9, v13, v14
-; VI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v7
-; VI-NEXT: v_div_fixup_f32 v9, v9, v6, 1.0
-; VI-NEXT: s_cbranch_vccnz .LBB9_17
-; VI-NEXT: ; %bb.14: ; %frem.loop_body.preheader
-; VI-NEXT: v_sub_u32_e32 v7, vcc, v10, v11
-; VI-NEXT: v_add_u32_e32 v7, vcc, 11, v7
-; VI-NEXT: .LBB9_15: ; %frem.loop_body
+; VI-NEXT: s_cselect_b64 vcc, exec, 0
+; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_cmp_le_i32 s8, 11
+; VI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; VI-NEXT: s_cbranch_scc1 .LBB9_14
+; VI-NEXT: ; %bb.12: ; %frem.loop_body.preheader
+; VI-NEXT: s_add_i32 s10, s10, 11
+; VI-NEXT: s_sub_i32 s8, s10, s11
+; VI-NEXT: .LBB9_13: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v10, v8
-; VI-NEXT: v_mul_f32_e32 v8, v10, v9
-; VI-NEXT: v_rndne_f32_e32 v8, v8
-; VI-NEXT: v_fma_f32 v8, -v8, v6, v10
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v8
-; VI-NEXT: v_add_f32_e32 v11, v8, v6
-; VI-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
-; VI-NEXT: v_add_u32_e32 v7, vcc, -11, v7
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v7
-; VI-NEXT: v_ldexp_f32 v8, v8, 11
-; VI-NEXT: s_cbranch_vccnz .LBB9_15
-; VI-NEXT: ; %bb.16: ; %Flow
-; VI-NEXT: v_mov_b32_e32 v8, v10
-; VI-NEXT: .LBB9_17: ; %frem.loop_exit
-; VI-NEXT: v_add_u32_e32 v7, vcc, -10, v7
-; VI-NEXT: v_ldexp_f32 v7, v8, v7
-; VI-NEXT: v_mul_f32_e32 v8, v7, v9
-; VI-NEXT: v_rndne_f32_e32 v8, v8
-; VI-NEXT: v_fma_f32 v7, -v8, v6, v7
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v7
-; VI-NEXT: v_add_f32_e32 v6, v7, v6
-; VI-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
-; VI-NEXT: v_ldexp_f32 v5, v6, v5
-; VI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; VI-NEXT: s_movk_i32 s2, 0x7fff
-; VI-NEXT: v_bfi_b32 v5, s2, v5, v3
-; VI-NEXT: .LBB9_18: ; %Flow54
-; VI-NEXT: s_movk_i32 s4, 0x7c00
-; VI-NEXT: v_cmp_lg_f16_e32 vcc, 0, v1
-; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |v0|, s4
-; VI-NEXT: s_and_b64 vcc, s[2:3], vcc
-; VI-NEXT: v_mov_b32_e32 v6, 0x7e00
-; VI-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
+; VI-NEXT: v_mul_f32_e32 v2, s9, v1
+; VI-NEXT: v_rndne_f32_e32 v2, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s9
+; VI-NEXT: v_readfirstlane_b32 s10, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; VI-NEXT: v_add_f32_e32 v2, v2, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s11, v2
+; VI-NEXT: s_cselect_b32 s10, s11, s10
+; VI-NEXT: v_ldexp_f32 v2, s10, 11
+; VI-NEXT: v_readfirstlane_b32 s11, v2
+; VI-NEXT: s_add_i32 s8, s8, -11
+; VI-NEXT: s_cmp_gt_i32 s8, 11
+; VI-NEXT: s_mov_b32 s10, s9
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_cbranch_scc1 .LBB9_13
+; VI-NEXT: s_branch .LBB9_15
+; VI-NEXT: .LBB9_14:
+; VI-NEXT: s_mov_b32 s10, s9
+; VI-NEXT: .LBB9_15: ; %frem.loop_exit
+; VI-NEXT: s_add_i32 s8, s8, -10
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_ldexp_f32 v2, s10, v2
+; VI-NEXT: v_mul_f32_e32 v1, v2, v1
+; VI-NEXT: v_rndne_f32_e32 v1, v1
+; VI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: v_readfirstlane_b32 s8, v1
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s9, v0
+; VI-NEXT: s_cselect_b32 s8, s9, s8
+; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: v_ldexp_f32 v0, s8, v0
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: s_and_b32 s8, s5, 0xffff8000
+; VI-NEXT: v_readfirstlane_b32 s7, v0
+; VI-NEXT: s_and_b32 s7, s7, 0x7fff
+; VI-NEXT: s_or_b32 s7, s7, s8
+; VI-NEXT: .LBB9_16: ; %Flow54
+; VI-NEXT: v_cmp_nlg_f16_e64 s[8:9], s3, 0
+; VI-NEXT: v_mov_b32_e32 v0, 0x7c00
+; VI-NEXT: s_cmp_lg_u64 s[8:9], 0
+; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |s2|, v0
+; VI-NEXT: s_cselect_b32 s8, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: v_cmp_nlg_f16_e64 s[2:3], s6, 0
+; VI-NEXT: s_cselect_b32 s10, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |s5|, v0
+; VI-NEXT: s_movk_i32 s9, 0x7e00
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: s_cselect_b32 s2, 1, 0
+; VI-NEXT: s_bfe_u32 s3, s9, 0x100000
+; VI-NEXT: s_bfe_u32 s4, s4, 0x100000
+; VI-NEXT: s_cmp_lg_u32 s8, 0
+; VI-NEXT: s_cselect_b32 s4, s3, s4
+; VI-NEXT: s_cmp_lg_u32 s10, 0
+; VI-NEXT: s_cselect_b32 s4, s4, s3
+; VI-NEXT: s_bfe_u32 s5, s7, 0x100000
+; VI-NEXT: s_cmp_lg_u32 s6, 0
+; VI-NEXT: s_cselect_b32 s5, s3, s5
+; VI-NEXT: s_cmp_lg_u32 s2, 0
+; VI-NEXT: s_cselect_b32 s2, s5, s3
+; VI-NEXT: s_lshl_b32 s2, s2, 16
+; VI-NEXT: s_or_b32 s2, s4, s2
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_cmp_lg_f16_e32 vcc, 0, v4
-; VI-NEXT: v_cmp_nge_f16_e64 s[0:1], |v3|, s4
-; VI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; VI-NEXT: v_cndmask_b32_sdwa v3, v6, v5, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%gep2 = getelementptr <2 x half>, ptr addrspace(1) %in2, i32 4
@@ -1522,725 +1642,886 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %in1, ptr addrspace(1) %in2) #0 {
; CI-LABEL: frem_v4f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: s_mov_b32 s6, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s0, s10
-; CI-NEXT: s_mov_b32 s1, s11
-; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[0:3], 0
-; CI-NEXT: s_mov_b32 s7, s3
-; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: v_readfirstlane_b32 s2, v1
-; CI-NEXT: v_readfirstlane_b32 s3, v0
-; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:32
-; CI-NEXT: v_cvt_f32_f16_e64 v3, |s3|
-; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e64 v2, |v0|
-; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v3, v2
-; CI-NEXT: s_cbranch_vccz .LBB10_2
+; CI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s4|
+; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[6:7], vcc, vcc
+; CI-NEXT: s_mov_b32 s7, 1
+; CI-NEXT: ; implicit-def: $sgpr6
+; CI-NEXT: s_cbranch_scc0 .LBB10_2
; CI-NEXT: ; %bb.1: ; %frem.else86
-; CI-NEXT: s_and_b32 s4, s3, 0xffff8000
-; CI-NEXT: v_cmp_eq_f32_e32 vcc, v3, v2
-; CI-NEXT: s_and_b64 s[0:1], vcc, exec
-; CI-NEXT: s_cselect_b32 s4, s4, s3
-; CI-NEXT: s_mov_b64 s[0:1], 0
-; CI-NEXT: s_branch .LBB10_3
-; CI-NEXT: .LBB10_2:
-; CI-NEXT: s_mov_b64 s[0:1], -1
-; CI-NEXT: ; implicit-def: $sgpr4
-; CI-NEXT: .LBB10_3: ; %Flow135
-; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: s_cselect_b32 s0, 1, 0
-; CI-NEXT: s_cmp_lg_u32 s0, 1
-; CI-NEXT: s_cbranch_scc1 .LBB10_9
-; CI-NEXT: ; %bb.4: ; %frem.compute85
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v7, v3
-; CI-NEXT: v_frexp_mant_f32_e32 v3, v3
-; CI-NEXT: v_ldexp_f32_e64 v5, v3, 11
-; CI-NEXT: v_frexp_mant_f32_e32 v3, v2
-; CI-NEXT: v_ldexp_f32_e64 v3, v3, 1
-; CI-NEXT: v_div_scale_f32 v9, s[0:1], v3, v3, 1.0
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v8, v2
-; CI-NEXT: v_add_i32_e32 v2, vcc, -1, v8
-; CI-NEXT: v_not_b32_e32 v4, v2
-; CI-NEXT: v_add_i32_e32 v4, vcc, v4, v7
-; CI-NEXT: v_div_scale_f32 v6, vcc, 1.0, v3, 1.0
-; CI-NEXT: v_rcp_f32_e32 v10, v9
+; CI-NEXT: s_and_b32 s6, s2, 0xffff8000
+; CI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
+; CI-NEXT: s_mov_b32 s7, 0
+; CI-NEXT: s_cselect_b32 s6, s6, s2
+; CI-NEXT: .LBB10_2: ; %Flow135
+; CI-NEXT: s_xor_b32 s7, s7, 1
+; CI-NEXT: s_cmp_lg_u32 s7, 0
+; CI-NEXT: s_cbranch_scc1 .LBB10_8
+; CI-NEXT: ; %bb.3: ; %frem.compute85
+; CI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; CI-NEXT: v_readfirstlane_b32 s9, v1
+; CI-NEXT: v_ldexp_f32_e64 v1, v2, 11
+; CI-NEXT: v_readfirstlane_b32 s8, v1
+; CI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; CI-NEXT: v_readfirstlane_b32 s10, v0
+; CI-NEXT: v_ldexp_f32_e64 v0, v1, 1
+; CI-NEXT: v_div_scale_f32 v1, s[12:13], v0, v0, 1.0
+; CI-NEXT: s_add_i32 s7, s9, -1
+; CI-NEXT: s_add_i32 s6, s10, -1
+; CI-NEXT: s_sub_i32 s7, s7, s6
+; CI-NEXT: v_div_scale_f32 v2, s[12:13], 1.0, v0, 1.0
+; CI-NEXT: s_or_b64 s[12:13], s[12:13], s[12:13]
+; CI-NEXT: v_rcp_f32_e32 v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v11, -v9, v10, 1.0
-; CI-NEXT: v_fma_f32 v10, v11, v10, v10
-; CI-NEXT: v_mul_f32_e32 v11, v6, v10
-; CI-NEXT: v_fma_f32 v12, -v9, v11, v6
-; CI-NEXT: v_fma_f32 v11, v12, v10, v11
-; CI-NEXT: v_fma_f32 v6, -v9, v11, v6
+; CI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; CI-NEXT: v_fma_f32 v3, v4, v3, v3
+; CI-NEXT: v_mul_f32_e32 v4, v2, v3
+; CI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; CI-NEXT: v_fma_f32 v4, v5, v3, v4
+; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v6, v6, v10, v11
-; CI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v4
-; CI-NEXT: v_div_fixup_f32 v6, v6, v3, 1.0
-; CI-NEXT: s_cbranch_vccnz .LBB10_8
-; CI-NEXT: ; %bb.5: ; %frem.loop_body93.preheader
-; CI-NEXT: v_sub_i32_e32 v4, vcc, v7, v8
-; CI-NEXT: v_add_i32_e32 v4, vcc, 11, v4
-; CI-NEXT: .LBB10_6: ; %frem.loop_body93
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_cmp_le_i32 s7, 11
+; CI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; CI-NEXT: s_cbranch_scc1 .LBB10_6
+; CI-NEXT: ; %bb.4: ; %frem.loop_body93.preheader
+; CI-NEXT: s_add_i32 s9, s9, 11
+; CI-NEXT: s_sub_i32 s7, s9, s10
+; CI-NEXT: .LBB10_5: ; %frem.loop_body93
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: v_mov_b32_e32 v7, v5
-; CI-NEXT: v_mul_f32_e32 v5, v7, v6
-; CI-NEXT: v_rndne_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v5, -v5, v3, v7
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v5
-; CI-NEXT: v_add_f32_e32 v8, v5, v3
-; CI-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc
-; CI-NEXT: v_add_i32_e32 v4, vcc, -11, v4
-; CI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v4
-; CI-NEXT: v_ldexp_f32_e64 v5, v5, 11
-; CI-NEXT: s_cbranch_vccnz .LBB10_6
-; CI-NEXT: ; %bb.7: ; %Flow133
-; CI-NEXT: v_mov_b32_e32 v5, v7
-; CI-NEXT: .LBB10_8: ; %frem.loop_exit94
-; CI-NEXT: v_add_i32_e32 v4, vcc, -10, v4
-; CI-NEXT: v_ldexp_f32_e32 v4, v5, v4
-; CI-NEXT: v_mul_f32_e32 v5, v4, v6
-; CI-NEXT: v_rndne_f32_e32 v5, v5
-; CI-NEXT: v_fma_f32 v4, -v5, v3, v4
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v4
-; CI-NEXT: v_add_f32_e32 v3, v4, v3
-; CI-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; CI-NEXT: v_ldexp_f32_e32 v2, v3, v2
-; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; CI-NEXT: s_and_b32 s0, s3, 0xffff8000
-; CI-NEXT: v_and_b32_e32 v2, 0x7fff, v2
-; CI-NEXT: v_or_b32_e32 v3, s0, v2
-; CI-NEXT: s_branch .LBB10_10
-; CI-NEXT: .LBB10_9:
-; CI-NEXT: v_mov_b32_e32 v3, s4
-; CI-NEXT: .LBB10_10: ; %Flow136
-; CI-NEXT: s_lshr_b32 s4, s2, 16
-; CI-NEXT: s_lshr_b32 s5, s3, 16
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; CI-NEXT: ; %bb.11:
-; CI-NEXT: v_cvt_f32_f16_e64 v6, |s5|
-; CI-NEXT: v_cvt_f32_f16_e64 v5, |v4|
-; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v6, v5
-; CI-NEXT: s_cbranch_vccz .LBB10_13
-; CI-NEXT: ; %bb.12: ; %frem.else53
-; CI-NEXT: s_and_b32 s6, s5, 0x8000
-; CI-NEXT: v_cmp_eq_f32_e32 vcc, v6, v5
-; CI-NEXT: s_and_b64 s[0:1], vcc, exec
-; CI-NEXT: s_cselect_b32 s6, s6, s5
-; CI-NEXT: s_mov_b64 s[0:1], 0
-; CI-NEXT: s_branch .LBB10_14
-; CI-NEXT: .LBB10_13:
-; CI-NEXT: s_mov_b64 s[0:1], -1
-; CI-NEXT: ; implicit-def: $sgpr6
-; CI-NEXT: .LBB10_14: ; %Flow131
-; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: s_cselect_b32 s0, 1, 0
-; CI-NEXT: s_cmp_lg_u32 s0, 1
-; CI-NEXT: s_cbranch_scc1 .LBB10_20
-; CI-NEXT: ; %bb.15: ; %frem.compute52
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v10, v6
-; CI-NEXT: v_frexp_mant_f32_e32 v6, v6
-; CI-NEXT: v_ldexp_f32_e64 v8, v6, 11
-; CI-NEXT: v_frexp_mant_f32_e32 v6, v5
-; CI-NEXT: v_ldexp_f32_e64 v6, v6, 1
-; CI-NEXT: v_div_scale_f32 v12, s[0:1], v6, v6, 1.0
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v11, v5
-; CI-NEXT: v_add_i32_e32 v5, vcc, -1, v11
-; CI-NEXT: v_not_b32_e32 v7, v5
-; CI-NEXT: v_add_i32_e32 v7, vcc, v7, v10
-; CI-NEXT: v_div_scale_f32 v9, vcc, 1.0, v6, 1.0
-; CI-NEXT: v_rcp_f32_e32 v13, v12
+; CI-NEXT: v_mul_f32_e32 v2, s8, v1
+; CI-NEXT: v_rndne_f32_e32 v2, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s8
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s9, v2
+; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v2, v2, v0
+; CI-NEXT: v_readfirstlane_b32 s10, v2
+; CI-NEXT: s_cselect_b32 s9, s10, s9
+; CI-NEXT: v_ldexp_f32_e64 v2, s9, 11
+; CI-NEXT: v_readfirstlane_b32 s10, v2
+; CI-NEXT: s_add_i32 s7, s7, -11
+; CI-NEXT: s_cmp_gt_i32 s7, 11
+; CI-NEXT: s_mov_b32 s9, s8
+; CI-NEXT: s_mov_b32 s8, s10
+; CI-NEXT: s_cbranch_scc1 .LBB10_5
+; CI-NEXT: s_branch .LBB10_7
+; CI-NEXT: .LBB10_6:
+; CI-NEXT: s_mov_b32 s9, s8
+; CI-NEXT: .LBB10_7: ; %frem.loop_exit94
+; CI-NEXT: s_add_i32 s7, s7, -10
+; CI-NEXT: v_mov_b32_e32 v2, s7
+; CI-NEXT: v_ldexp_f32_e32 v2, s9, v2
+; CI-NEXT: v_mul_f32_e32 v1, v2, v1
+; CI-NEXT: v_rndne_f32_e32 v1, v1
+; CI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; CI-NEXT: s_or_b64 s[8:9], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v0, v1, v0
+; CI-NEXT: v_readfirstlane_b32 s7, v1
+; CI-NEXT: v_readfirstlane_b32 s8, v0
+; CI-NEXT: s_cselect_b32 s7, s8, s7
+; CI-NEXT: v_mov_b32_e32 v0, s6
+; CI-NEXT: v_ldexp_f32_e32 v0, s7, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CI-NEXT: s_and_b32 s7, s2, 0xffff8000
+; CI-NEXT: v_readfirstlane_b32 s6, v0
+; CI-NEXT: s_and_b32 s6, s6, 0x7fff
+; CI-NEXT: s_or_b32 s6, s6, s7
+; CI-NEXT: .LBB10_8: ; %Flow136
+; CI-NEXT: s_lshr_b32 s7, s2, 16
+; CI-NEXT: s_lshr_b32 s8, s4, 16
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s7|
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s8|
+; CI-NEXT: ; implicit-def: $sgpr9
+; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
+; CI-NEXT: s_mov_b32 s10, 1
+; CI-NEXT: s_cbranch_scc0 .LBB10_10
+; CI-NEXT: ; %bb.9: ; %frem.else53
+; CI-NEXT: s_and_b32 s9, s7, 0xffff8000
+; CI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[12:13], vcc, vcc
+; CI-NEXT: s_mov_b32 s10, 0
+; CI-NEXT: s_cselect_b32 s9, s9, s7
+; CI-NEXT: .LBB10_10: ; %Flow131
+; CI-NEXT: s_xor_b32 s10, s10, 1
+; CI-NEXT: s_cmp_lg_u32 s10, 0
+; CI-NEXT: s_cbranch_scc1 .LBB10_16
+; CI-NEXT: ; %bb.11: ; %frem.compute52
+; CI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; CI-NEXT: v_readfirstlane_b32 s12, v1
+; CI-NEXT: v_ldexp_f32_e64 v1, v2, 11
+; CI-NEXT: v_readfirstlane_b32 s11, v1
+; CI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; CI-NEXT: v_readfirstlane_b32 s13, v0
+; CI-NEXT: v_ldexp_f32_e64 v0, v1, 1
+; CI-NEXT: v_div_scale_f32 v1, s[14:15], v0, v0, 1.0
+; CI-NEXT: s_add_i32 s10, s12, -1
+; CI-NEXT: s_add_i32 s9, s13, -1
+; CI-NEXT: s_sub_i32 s10, s10, s9
+; CI-NEXT: v_div_scale_f32 v2, s[14:15], 1.0, v0, 1.0
+; CI-NEXT: s_or_b64 s[14:15], s[14:15], s[14:15]
+; CI-NEXT: v_rcp_f32_e32 v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v14, -v12, v13, 1.0
-; CI-NEXT: v_fma_f32 v13, v14, v13, v13
-; CI-NEXT: v_mul_f32_e32 v14, v9, v13
-; CI-NEXT: v_fma_f32 v15, -v12, v14, v9
-; CI-NEXT: v_fma_f32 v14, v15, v13, v14
-; CI-NEXT: v_fma_f32 v9, -v12, v14, v9
+; CI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; CI-NEXT: v_fma_f32 v3, v4, v3, v3
+; CI-NEXT: v_mul_f32_e32 v4, v2, v3
+; CI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; CI-NEXT: v_fma_f32 v4, v5, v3, v4
+; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v9, v9, v13, v14
-; CI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v7
-; CI-NEXT: v_div_fixup_f32 v9, v9, v6, 1.0
-; CI-NEXT: s_cbranch_vccnz .LBB10_19
-; CI-NEXT: ; %bb.16: ; %frem.loop_body60.preheader
-; CI-NEXT: v_sub_i32_e32 v7, vcc, v10, v11
-; CI-NEXT: v_add_i32_e32 v7, vcc, 11, v7
-; CI-NEXT: .LBB10_17: ; %frem.loop_body60
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_cmp_le_i32 s10, 11
+; CI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; CI-NEXT: s_cbranch_scc1 .LBB10_14
+; CI-NEXT: ; %bb.12: ; %frem.loop_body60.preheader
+; CI-NEXT: s_add_i32 s12, s12, 11
+; CI-NEXT: s_sub_i32 s10, s12, s13
+; CI-NEXT: .LBB10_13: ; %frem.loop_body60
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: v_mov_b32_e32 v10, v8
-; CI-NEXT: v_mul_f32_e32 v8, v10, v9
-; CI-NEXT: v_rndne_f32_e32 v8, v8
-; CI-NEXT: v_fma_f32 v8, -v8, v6, v10
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v8
-; CI-NEXT: v_add_f32_e32 v11, v8, v6
-; CI-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
-; CI-NEXT: v_add_i32_e32 v7, vcc, -11, v7
-; CI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v7
-; CI-NEXT: v_ldexp_f32_e64 v8, v8, 11
-; CI-NEXT: s_cbranch_vccnz .LBB10_17
-; CI-NEXT: ; %bb.18: ; %Flow129
-; CI-NEXT: v_mov_b32_e32 v8, v10
-; CI-NEXT: .LBB10_19: ; %frem.loop_exit61
-; CI-NEXT: v_add_i32_e32 v7, vcc, -10, v7
-; CI-NEXT: v_ldexp_f32_e32 v7, v8, v7
-; CI-NEXT: v_mul_f32_e32 v8, v7, v9
-; CI-NEXT: v_rndne_f32_e32 v8, v8
-; CI-NEXT: v_fma_f32 v7, -v8, v6, v7
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v7
-; CI-NEXT: v_add_f32_e32 v6, v7, v6
-; CI-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
-; CI-NEXT: v_ldexp_f32_e32 v5, v6, v5
-; CI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; CI-NEXT: s_and_b32 s0, s5, 0x8000
-; CI-NEXT: v_and_b32_e32 v5, 0x7fff, v5
-; CI-NEXT: v_or_b32_e32 v5, s0, v5
-; CI-NEXT: s_branch .LBB10_21
-; CI-NEXT: .LBB10_20:
-; CI-NEXT: v_mov_b32_e32 v5, s6
-; CI-NEXT: .LBB10_21:
-; CI-NEXT: v_cvt_f32_f16_e64 v7, |s2|
-; CI-NEXT: v_cvt_f32_f16_e64 v6, |v1|
-; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v7, v6
-; CI-NEXT: s_cbranch_vccz .LBB10_23
-; CI-NEXT: ; %bb.22: ; %frem.else20
-; CI-NEXT: s_and_b32 s6, s2, 0xffff8000
-; CI-NEXT: v_cmp_eq_f32_e32 vcc, v7, v6
-; CI-NEXT: s_and_b64 s[0:1], vcc, exec
-; CI-NEXT: s_cselect_b32 s6, s6, s2
-; CI-NEXT: s_mov_b64 s[0:1], 0
-; CI-NEXT: s_branch .LBB10_24
-; CI-NEXT: .LBB10_23:
-; CI-NEXT: s_mov_b64 s[0:1], -1
-; CI-NEXT: ; implicit-def: $sgpr6
-; CI-NEXT: .LBB10_24: ; %Flow127
-; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: s_cselect_b32 s0, 1, 0
-; CI-NEXT: s_cmp_lg_u32 s0, 1
-; CI-NEXT: s_cbranch_scc1 .LBB10_30
-; CI-NEXT: ; %bb.25: ; %frem.compute19
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v11, v7
-; CI-NEXT: v_frexp_mant_f32_e32 v7, v7
-; CI-NEXT: v_ldexp_f32_e64 v9, v7, 11
-; CI-NEXT: v_frexp_mant_f32_e32 v7, v6
-; CI-NEXT: v_ldexp_f32_e64 v7, v7, 1
-; CI-NEXT: v_div_scale_f32 v13, s[0:1], v7, v7, 1.0
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v12, v6
-; CI-NEXT: v_add_i32_e32 v6, vcc, -1, v12
-; CI-NEXT: v_not_b32_e32 v8, v6
-; CI-NEXT: v_add_i32_e32 v8, vcc, v8, v11
-; CI-NEXT: v_div_scale_f32 v10, vcc, 1.0, v7, 1.0
-; CI-NEXT: v_rcp_f32_e32 v14, v13
+; CI-NEXT: v_mul_f32_e32 v2, s11, v1
+; CI-NEXT: v_rndne_f32_e32 v2, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s11
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s14, v2
+; CI-NEXT: s_or_b64 s[12:13], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v2, v2, v0
+; CI-NEXT: v_readfirstlane_b32 s12, v2
+; CI-NEXT: s_cselect_b32 s12, s12, s14
+; CI-NEXT: v_ldexp_f32_e64 v2, s12, 11
+; CI-NEXT: v_readfirstlane_b32 s13, v2
+; CI-NEXT: s_add_i32 s10, s10, -11
+; CI-NEXT: s_cmp_gt_i32 s10, 11
+; CI-NEXT: s_mov_b32 s12, s11
+; CI-NEXT: s_mov_b32 s11, s13
+; CI-NEXT: s_cbranch_scc1 .LBB10_13
+; CI-NEXT: s_branch .LBB10_15
+; CI-NEXT: .LBB10_14:
+; CI-NEXT: s_mov_b32 s12, s11
+; CI-NEXT: .LBB10_15: ; %frem.loop_exit61
+; CI-NEXT: s_add_i32 s10, s10, -10
+; CI-NEXT: v_mov_b32_e32 v2, s10
+; CI-NEXT: v_ldexp_f32_e32 v2, s12, v2
+; CI-NEXT: v_mul_f32_e32 v1, v2, v1
+; CI-NEXT: v_rndne_f32_e32 v1, v1
+; CI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v0, v1, v0
+; CI-NEXT: v_readfirstlane_b32 s12, v1
+; CI-NEXT: v_readfirstlane_b32 s10, v0
+; CI-NEXT: s_cselect_b32 s10, s10, s12
+; CI-NEXT: v_mov_b32_e32 v0, s9
+; CI-NEXT: v_ldexp_f32_e32 v0, s10, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CI-NEXT: s_and_b32 s10, s7, 0xffff8000
+; CI-NEXT: v_readfirstlane_b32 s9, v0
+; CI-NEXT: s_and_b32 s9, s9, 0x7fff
+; CI-NEXT: s_or_b32 s9, s9, s10
+; CI-NEXT: .LBB10_16: ; %Flow132
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s3|
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s5|
+; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
+; CI-NEXT: s_mov_b32 s11, 1
+; CI-NEXT: ; implicit-def: $sgpr10
+; CI-NEXT: s_cbranch_scc0 .LBB10_18
+; CI-NEXT: ; %bb.17: ; %frem.else20
+; CI-NEXT: s_and_b32 s10, s3, 0xffff8000
+; CI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[12:13], vcc, vcc
+; CI-NEXT: s_mov_b32 s11, 0
+; CI-NEXT: s_cselect_b32 s10, s10, s3
+; CI-NEXT: .LBB10_18: ; %Flow127
+; CI-NEXT: s_xor_b32 s11, s11, 1
+; CI-NEXT: s_cmp_lg_u32 s11, 0
+; CI-NEXT: s_cbranch_scc1 .LBB10_24
+; CI-NEXT: ; %bb.19: ; %frem.compute19
+; CI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; CI-NEXT: v_readfirstlane_b32 s13, v1
+; CI-NEXT: v_ldexp_f32_e64 v1, v2, 11
+; CI-NEXT: v_readfirstlane_b32 s12, v1
+; CI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; CI-NEXT: v_readfirstlane_b32 s14, v0
+; CI-NEXT: v_ldexp_f32_e64 v0, v1, 1
+; CI-NEXT: v_div_scale_f32 v1, s[16:17], v0, v0, 1.0
+; CI-NEXT: s_add_i32 s11, s13, -1
+; CI-NEXT: s_add_i32 s10, s14, -1
+; CI-NEXT: s_sub_i32 s11, s11, s10
+; CI-NEXT: v_div_scale_f32 v2, s[16:17], 1.0, v0, 1.0
+; CI-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
+; CI-NEXT: v_rcp_f32_e32 v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v15, -v13, v14, 1.0
-; CI-NEXT: v_fma_f32 v14, v15, v14, v14
-; CI-NEXT: v_mul_f32_e32 v15, v10, v14
-; CI-NEXT: v_fma_f32 v16, -v13, v15, v10
-; CI-NEXT: v_fma_f32 v15, v16, v14, v15
-; CI-NEXT: v_fma_f32 v10, -v13, v15, v10
+; CI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; CI-NEXT: v_fma_f32 v3, v4, v3, v3
+; CI-NEXT: v_mul_f32_e32 v4, v2, v3
+; CI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; CI-NEXT: v_fma_f32 v4, v5, v3, v4
+; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v10, v10, v14, v15
-; CI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v8
-; CI-NEXT: v_div_fixup_f32 v10, v10, v7, 1.0
-; CI-NEXT: s_cbranch_vccnz .LBB10_29
-; CI-NEXT: ; %bb.26: ; %frem.loop_body27.preheader
-; CI-NEXT: v_sub_i32_e32 v8, vcc, v11, v12
-; CI-NEXT: v_add_i32_e32 v8, vcc, 11, v8
-; CI-NEXT: .LBB10_27: ; %frem.loop_body27
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_cmp_le_i32 s11, 11
+; CI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; CI-NEXT: s_cbranch_scc1 .LBB10_22
+; CI-NEXT: ; %bb.20: ; %frem.loop_body27.preheader
+; CI-NEXT: s_add_i32 s13, s13, 11
+; CI-NEXT: s_sub_i32 s11, s13, s14
+; CI-NEXT: .LBB10_21: ; %frem.loop_body27
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: v_mov_b32_e32 v11, v9
-; CI-NEXT: v_mul_f32_e32 v9, v11, v10
-; CI-NEXT: v_rndne_f32_e32 v9, v9
-; CI-NEXT: v_fma_f32 v9, -v9, v7, v11
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v9
-; CI-NEXT: v_add_f32_e32 v12, v9, v7
-; CI-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc
-; CI-NEXT: v_add_i32_e32 v8, vcc, -11, v8
-; CI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v8
-; CI-NEXT: v_ldexp_f32_e64 v9, v9, 11
-; CI-NEXT: s_cbranch_vccnz .LBB10_27
-; CI-NEXT: ; %bb.28: ; %Flow125
-; CI-NEXT: v_mov_b32_e32 v9, v11
-; CI-NEXT: .LBB10_29: ; %frem.loop_exit28
-; CI-NEXT: v_add_i32_e32 v8, vcc, -10, v8
-; CI-NEXT: v_ldexp_f32_e32 v8, v9, v8
-; CI-NEXT: v_mul_f32_e32 v9, v8, v10
-; CI-NEXT: v_rndne_f32_e32 v9, v9
-; CI-NEXT: v_fma_f32 v8, -v9, v7, v8
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v8
-; CI-NEXT: v_add_f32_e32 v7, v8, v7
-; CI-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc
-; CI-NEXT: v_ldexp_f32_e32 v6, v7, v6
-; CI-NEXT: v_cvt_f16_f32_e32 v6, v6
-; CI-NEXT: s_and_b32 s0, s2, 0xffff8000
-; CI-NEXT: v_and_b32_e32 v6, 0x7fff, v6
-; CI-NEXT: v_or_b32_e32 v6, s0, v6
-; CI-NEXT: s_branch .LBB10_31
-; CI-NEXT: .LBB10_30:
-; CI-NEXT: v_mov_b32_e32 v6, s6
-; CI-NEXT: .LBB10_31:
-; CI-NEXT: v_cvt_f32_f16_e64 v8, |s4|
-; CI-NEXT: v_cvt_f32_f16_e64 v7, |v2|
-; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v8, v7
-; CI-NEXT: s_cbranch_vccz .LBB10_33
-; CI-NEXT: ; %bb.32: ; %frem.else
-; CI-NEXT: s_and_b32 s6, s4, 0x8000
-; CI-NEXT: v_cmp_eq_f32_e32 vcc, v8, v7
-; CI-NEXT: s_and_b64 s[0:1], vcc, exec
-; CI-NEXT: s_cselect_b32 s6, s6, s4
-; CI-NEXT: s_mov_b64 s[0:1], 0
-; CI-NEXT: s_branch .LBB10_34
-; CI-NEXT: .LBB10_33:
-; CI-NEXT: s_mov_b64 s[0:1], -1
-; CI-NEXT: ; implicit-def: $sgpr6
-; CI-NEXT: .LBB10_34: ; %Flow123
-; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: s_cselect_b32 s0, 1, 0
-; CI-NEXT: s_cmp_lg_u32 s0, 1
-; CI-NEXT: s_cbranch_scc1 .LBB10_40
-; CI-NEXT: ; %bb.35: ; %frem.compute
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v12, v8
-; CI-NEXT: v_frexp_mant_f32_e32 v8, v8
-; CI-NEXT: v_ldexp_f32_e64 v10, v8, 11
-; CI-NEXT: v_frexp_mant_f32_e32 v8, v7
-; CI-NEXT: v_ldexp_f32_e64 v8, v8, 1
-; CI-NEXT: v_div_scale_f32 v14, s[0:1], v8, v8, 1.0
-; CI-NEXT: v_frexp_exp_i32_f32_e32 v13, v7
-; CI-NEXT: v_add_i32_e32 v7, vcc, -1, v13
-; CI-NEXT: v_not_b32_e32 v9, v7
-; CI-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; CI-NEXT: v_div_scale_f32 v11, vcc, 1.0, v8, 1.0
-; CI-NEXT: v_rcp_f32_e32 v15, v14
+; CI-NEXT: v_mul_f32_e32 v2, s12, v1
+; CI-NEXT: v_rndne_f32_e32 v2, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s12
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s13, v2
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v2, v2, v0
+; CI-NEXT: v_readfirstlane_b32 s14, v2
+; CI-NEXT: s_cselect_b32 s13, s14, s13
+; CI-NEXT: v_ldexp_f32_e64 v2, s13, 11
+; CI-NEXT: v_readfirstlane_b32 s14, v2
+; CI-NEXT: s_add_i32 s11, s11, -11
+; CI-NEXT: s_cmp_gt_i32 s11, 11
+; CI-NEXT: s_mov_b32 s13, s12
+; CI-NEXT: s_mov_b32 s12, s14
+; CI-NEXT: s_cbranch_scc1 .LBB10_21
+; CI-NEXT: s_branch .LBB10_23
+; CI-NEXT: .LBB10_22:
+; CI-NEXT: s_mov_b32 s13, s12
+; CI-NEXT: .LBB10_23: ; %frem.loop_exit28
+; CI-NEXT: s_add_i32 s11, s11, -10
+; CI-NEXT: v_mov_b32_e32 v2, s11
+; CI-NEXT: v_ldexp_f32_e32 v2, s13, v2
+; CI-NEXT: v_mul_f32_e32 v1, v2, v1
+; CI-NEXT: v_rndne_f32_e32 v1, v1
+; CI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; CI-NEXT: s_or_b64 s[12:13], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v0, v1, v0
+; CI-NEXT: v_readfirstlane_b32 s11, v1
+; CI-NEXT: v_readfirstlane_b32 s12, v0
+; CI-NEXT: s_cselect_b32 s11, s12, s11
+; CI-NEXT: v_mov_b32_e32 v0, s10
+; CI-NEXT: v_ldexp_f32_e32 v0, s11, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CI-NEXT: s_and_b32 s11, s3, 0xffff8000
+; CI-NEXT: v_readfirstlane_b32 s10, v0
+; CI-NEXT: s_and_b32 s10, s10, 0x7fff
+; CI-NEXT: s_or_b32 s10, s10, s11
+; CI-NEXT: .LBB10_24: ; %Flow128
+; CI-NEXT: s_lshr_b32 s11, s3, 16
+; CI-NEXT: s_lshr_b32 s12, s5, 16
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s11|
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s12|
+; CI-NEXT: ; implicit-def: $sgpr13
+; CI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
+; CI-NEXT: s_mov_b32 s14, 1
+; CI-NEXT: s_cbranch_scc0 .LBB10_26
+; CI-NEXT: ; %bb.25: ; %frem.else
+; CI-NEXT: s_and_b32 s13, s11, 0xffff8000
+; CI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; CI-NEXT: s_or_b64 s[16:17], vcc, vcc
+; CI-NEXT: s_mov_b32 s14, 0
+; CI-NEXT: s_cselect_b32 s13, s13, s11
+; CI-NEXT: .LBB10_26: ; %Flow123
+; CI-NEXT: s_xor_b32 s14, s14, 1
+; CI-NEXT: s_cmp_lg_u32 s14, 0
+; CI-NEXT: s_cbranch_scc1 .LBB10_32
+; CI-NEXT: ; %bb.27: ; %frem.compute
+; CI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; CI-NEXT: v_readfirstlane_b32 s16, v1
+; CI-NEXT: v_ldexp_f32_e64 v1, v2, 11
+; CI-NEXT: v_readfirstlane_b32 s15, v1
+; CI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; CI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; CI-NEXT: v_readfirstlane_b32 s17, v0
+; CI-NEXT: v_ldexp_f32_e64 v0, v1, 1
+; CI-NEXT: v_div_scale_f32 v1, s[18:19], v0, v0, 1.0
+; CI-NEXT: s_add_i32 s14, s16, -1
+; CI-NEXT: s_add_i32 s13, s17, -1
+; CI-NEXT: s_sub_i32 s14, s14, s13
+; CI-NEXT: v_div_scale_f32 v2, s[18:19], 1.0, v0, 1.0
+; CI-NEXT: s_or_b64 s[18:19], s[18:19], s[18:19]
+; CI-NEXT: v_rcp_f32_e32 v3, v1
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; CI-NEXT: v_fma_f32 v16, -v14, v15, 1.0
-; CI-NEXT: v_fma_f32 v15, v16, v15, v15
-; CI-NEXT: v_mul_f32_e32 v16, v11, v15
-; CI-NEXT: v_fma_f32 v17, -v14, v16, v11
-; CI-NEXT: v_fma_f32 v16, v17, v15, v16
-; CI-NEXT: v_fma_f32 v11, -v14, v16, v11
+; CI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; CI-NEXT: v_fma_f32 v3, v4, v3, v3
+; CI-NEXT: v_mul_f32_e32 v4, v2, v3
+; CI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; CI-NEXT: v_fma_f32 v4, v5, v3, v4
+; CI-NEXT: v_fma_f32 v1, -v1, v4, v2
; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; CI-NEXT: v_div_fmas_f32 v11, v11, v15, v16
-; CI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v9
-; CI-NEXT: v_div_fixup_f32 v11, v11, v8, 1.0
-; CI-NEXT: s_cbranch_vccnz .LBB10_39
-; CI-NEXT: ; %bb.36: ; %frem.loop_body.preheader
-; CI-NEXT: v_sub_i32_e32 v9, vcc, v12, v13
-; CI-NEXT: v_add_i32_e32 v9, vcc, 11, v9
-; CI-NEXT: .LBB10_37: ; %frem.loop_body
+; CI-NEXT: s_cselect_b64 vcc, exec, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; CI-NEXT: s_cmp_le_i32 s14, 11
+; CI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; CI-NEXT: s_cbranch_scc1 .LBB10_30
+; CI-NEXT: ; %bb.28: ; %frem.loop_body.preheader
+; CI-NEXT: s_add_i32 s16, s16, 11
+; CI-NEXT: s_sub_i32 s14, s16, s17
+; CI-NEXT: .LBB10_29: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: v_mov_b32_e32 v12, v10
-; CI-NEXT: v_mul_f32_e32 v10, v12, v11
-; CI-NEXT: v_rndne_f32_e32 v10, v10
-; CI-NEXT: v_fma_f32 v10, -v10, v8, v12
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v10
-; CI-NEXT: v_add_f32_e32 v13, v10, v8
-; CI-NEXT: v_cndmask_b32_e32 v10, v10, v13, vcc
-; CI-NEXT: v_add_i32_e32 v9, vcc, -11, v9
-; CI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v9
-; CI-NEXT: v_ldexp_f32_e64 v10, v10, 11
-; CI-NEXT: s_cbranch_vccnz .LBB10_37
-; CI-NEXT: ; %bb.38: ; %Flow
-; CI-NEXT: v_mov_b32_e32 v10, v12
-; CI-NEXT: .LBB10_39: ; %frem.loop_exit
-; CI-NEXT: v_add_i32_e32 v9, vcc, -10, v9
-; CI-NEXT: v_ldexp_f32_e32 v9, v10, v9
-; CI-NEXT: v_mul_f32_e32 v10, v9, v11
-; CI-NEXT: v_rndne_f32_e32 v10, v10
-; CI-NEXT: v_fma_f32 v9, -v10, v8, v9
-; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v9
-; CI-NEXT: v_add_f32_e32 v8, v9, v8
-; CI-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
-; CI-NEXT: v_ldexp_f32_e32 v7, v8, v7
-; CI-NEXT: v_cvt_f16_f32_e32 v7, v7
-; CI-NEXT: s_and_b32 s0, s4, 0x8000
-; CI-NEXT: v_and_b32_e32 v7, 0x7fff, v7
-; CI-NEXT: v_or_b32_e32 v7, s0, v7
-; CI-NEXT: s_branch .LBB10_41
-; CI-NEXT: .LBB10_40:
-; CI-NEXT: v_mov_b32_e32 v7, s6
-; CI-NEXT: .LBB10_41: ; %Flow124
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_cmp_lg_f32_e32 vcc, 0, v0
+; CI-NEXT: v_mul_f32_e32 v2, s15, v1
+; CI-NEXT: v_rndne_f32_e32 v2, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s15
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s18, v2
+; CI-NEXT: s_or_b64 s[16:17], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v2, v2, v0
+; CI-NEXT: v_readfirstlane_b32 s16, v2
+; CI-NEXT: s_cselect_b32 s16, s16, s18
+; CI-NEXT: v_ldexp_f32_e64 v2, s16, 11
+; CI-NEXT: v_readfirstlane_b32 s17, v2
+; CI-NEXT: s_add_i32 s14, s14, -11
+; CI-NEXT: s_cmp_gt_i32 s14, 11
+; CI-NEXT: s_mov_b32 s16, s15
+; CI-NEXT: s_mov_b32 s15, s17
+; CI-NEXT: s_cbranch_scc1 .LBB10_29
+; CI-NEXT: s_branch .LBB10_31
+; CI-NEXT: .LBB10_30:
+; CI-NEXT: s_mov_b32 s16, s15
+; CI-NEXT: .LBB10_31: ; %frem.loop_exit
+; CI-NEXT: s_add_i32 s14, s14, -10
+; CI-NEXT: v_mov_b32_e32 v2, s14
+; CI-NEXT: v_ldexp_f32_e32 v2, s16, v2
+; CI-NEXT: v_mul_f32_e32 v1, v2, v1
+; CI-NEXT: v_rndne_f32_e32 v1, v1
+; CI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
+; CI-NEXT: v_add_f32_e32 v0, v1, v0
+; CI-NEXT: v_readfirstlane_b32 s16, v1
+; CI-NEXT: v_readfirstlane_b32 s14, v0
+; CI-NEXT: s_cselect_b32 s14, s14, s16
+; CI-NEXT: v_mov_b32_e32 v0, s13
+; CI-NEXT: v_ldexp_f32_e32 v0, s14, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CI-NEXT: s_and_b32 s14, s11, 0xffff8000
+; CI-NEXT: v_readfirstlane_b32 s13, v0
+; CI-NEXT: s_and_b32 s13, s13, 0x7fff
+; CI-NEXT: s_or_b32 s13, s13, s14
+; CI-NEXT: .LBB10_32: ; %Flow124
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s4
+; CI-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; CI-NEXT: s_movk_i32 s17, 0x7e00
+; CI-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s2|
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
+; CI-NEXT: s_cselect_b32 s16, 1, 0
+; CI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s8
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
+; CI-NEXT: s_cselect_b32 s18, 1, 0
+; CI-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s7|
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
+; CI-NEXT: s_cselect_b32 s8, 1, 0
+; CI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s5
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
+; CI-NEXT: s_cselect_b32 s7, 1, 0
+; CI-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
; CI-NEXT: v_cvt_f32_f16_e64 v0, |s3|
-; CI-NEXT: s_mov_b32 s3, 0x7f800000
-; CI-NEXT: s_mov_b32 s11, 0xf000
-; CI-NEXT: s_mov_b32 s10, -1
-; CI-NEXT: v_cmp_nle_f32_e64 s[0:1], s3, v0
-; CI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; CI-NEXT: v_mov_b32_e32 v0, 0x7e00
-; CI-NEXT: v_cndmask_b32_e32 v3, v0, v3, vcc
-; CI-NEXT: v_cmp_lg_f32_e32 vcc, 0, v4
-; CI-NEXT: v_cvt_f32_f16_e64 v4, |s5|
-; CI-NEXT: v_cmp_nle_f32_e64 s[0:1], s3, v4
-; CI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; CI-NEXT: v_cndmask_b32_e32 v4, v0, v5, vcc
-; CI-NEXT: v_cmp_lg_f32_e32 vcc, 0, v1
-; CI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
-; CI-NEXT: v_cmp_nle_f32_e64 s[0:1], s3, v1
-; CI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; CI-NEXT: v_cndmask_b32_e32 v1, v0, v6, vcc
-; CI-NEXT: v_cmp_lg_f32_e32 vcc, 0, v2
-; CI-NEXT: v_cvt_f32_f16_e64 v2, |s4|
-; CI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; CI-NEXT: v_cmp_nle_f32_e64 s[0:1], s3, v2
-; CI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; CI-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
-; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_or_b32_e32 v1, v1, v0
-; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
-; CI-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; CI-NEXT: v_or_b32_e32 v0, v2, v0
-; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; CI-NEXT: s_or_b64 s[4:5], vcc, vcc
+; CI-NEXT: s_cselect_b32 s4, 1, 0
+; CI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s12
+; CI-NEXT: s_or_b64 s[2:3], vcc, vcc
+; CI-NEXT: s_cselect_b32 s5, 1, 0
+; CI-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s11|
+; CI-NEXT: s_or_b64 s[2:3], vcc, vcc
+; CI-NEXT: s_cselect_b32 s12, 1, 0
+; CI-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
+; CI-NEXT: s_or_b64 s[2:3], vcc, vcc
+; CI-NEXT: s_cselect_b32 s3, 1, 0
+; CI-NEXT: s_bfe_u32 s11, s17, 0x100000
+; CI-NEXT: s_bfe_u32 s2, s6, 0x100000
+; CI-NEXT: s_cmp_lg_u32 s16, 0
+; CI-NEXT: s_cselect_b32 s2, s11, s2
+; CI-NEXT: s_cmp_lg_u32 s18, 0
+; CI-NEXT: s_cselect_b32 s2, s2, s11
+; CI-NEXT: s_bfe_u32 s6, s9, 0x100000
+; CI-NEXT: s_cmp_lg_u32 s8, 0
+; CI-NEXT: s_cselect_b32 s6, s11, s6
+; CI-NEXT: s_cmp_lg_u32 s7, 0
+; CI-NEXT: s_cselect_b32 s6, s6, s11
+; CI-NEXT: s_lshl_b32 s6, s6, 16
+; CI-NEXT: s_or_b32 s2, s2, s6
+; CI-NEXT: s_bfe_u32 s6, s10, 0x100000
+; CI-NEXT: s_cmp_lg_u32 s4, 0
+; CI-NEXT: s_cselect_b32 s4, s11, s6
+; CI-NEXT: s_cmp_lg_u32 s5, 0
+; CI-NEXT: s_cselect_b32 s4, s4, s11
+; CI-NEXT: s_bfe_u32 s5, s13, 0x100000
+; CI-NEXT: s_cmp_lg_u32 s12, 0
+; CI-NEXT: s_cselect_b32 s5, s11, s5
+; CI-NEXT: s_cmp_lg_u32 s3, 0
+; CI-NEXT: s_cselect_b32 s3, s5, s11
+; CI-NEXT: s_lshl_b32 s3, s3, 16
+; CI-NEXT: s_or_b32 s3, s4, s3
+; CI-NEXT: v_mov_b32_e32 v0, s2
+; CI-NEXT: v_mov_b32_e32 v1, s3
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: frem_v4f16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT: s_mov_b32 s7, 1
+; VI-NEXT: ; implicit-def: $sgpr6
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: s_add_u32 s2, s4, 32
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: s_addc_u32 s3, s5, 0
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_f16_e64 v6, |v0|
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e64 v5, |v2|
-; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v6, v5
-; VI-NEXT: s_cbranch_vccz .LBB10_2
+; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x20
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
+; VI-NEXT: v_cvt_f32_f16_e64 v0, |s4|
+; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_cbranch_scc0 .LBB10_2
; VI-NEXT: ; %bb.1: ; %frem.else86
-; VI-NEXT: v_and_b32_e32 v4, 0x8000, v0
-; VI-NEXT: v_cmp_eq_f32_e32 vcc, v6, v5
-; VI-NEXT: v_cndmask_b32_e32 v4, v0, v4, vcc
-; VI-NEXT: s_mov_b64 s[2:3], 0
-; VI-NEXT: s_branch .LBB10_3
-; VI-NEXT: .LBB10_2:
-; VI-NEXT: s_mov_b64 s[2:3], -1
-; VI-NEXT: ; implicit-def: $vgpr4
-; VI-NEXT: .LBB10_3: ; %Flow135
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cbranch_scc1 .LBB10_9
-; VI-NEXT: ; %bb.4: ; %frem.compute85
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v9, v6
-; VI-NEXT: v_frexp_mant_f32_e32 v4, v6
-; VI-NEXT: v_frexp_mant_f32_e32 v6, v5
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v10, v5
-; VI-NEXT: v_ldexp_f32 v5, v6, 1
-; VI-NEXT: v_div_scale_f32 v11, s[2:3], v5, v5, 1.0
-; VI-NEXT: v_ldexp_f32 v7, v4, 11
-; VI-NEXT: v_add_u32_e32 v4, vcc, -1, v10
-; VI-NEXT: v_not_b32_e32 v6, v4
-; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v9
-; VI-NEXT: v_div_scale_f32 v8, vcc, 1.0, v5, 1.0
-; VI-NEXT: v_rcp_f32_e32 v12, v11
+; VI-NEXT: s_and_b32 s6, s2, 0xffff8000
+; VI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_mov_b32 s7, 0
+; VI-NEXT: s_cselect_b32 s6, s6, s2
+; VI-NEXT: .LBB10_2: ; %Flow135
+; VI-NEXT: s_xor_b32 s7, s7, 1
+; VI-NEXT: s_cmp_lg_u32 s7, 0
+; VI-NEXT: s_cbranch_scc1 .LBB10_8
+; VI-NEXT: ; %bb.3: ; %frem.compute85
+; VI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; VI-NEXT: v_readfirstlane_b32 s9, v1
+; VI-NEXT: v_ldexp_f32 v1, v2, 11
+; VI-NEXT: v_readfirstlane_b32 s8, v1
+; VI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; VI-NEXT: v_readfirstlane_b32 s10, v0
+; VI-NEXT: v_ldexp_f32 v0, v1, 1
+; VI-NEXT: v_div_scale_f32 v1, s[12:13], v0, v0, 1.0
+; VI-NEXT: s_add_i32 s7, s9, -1
+; VI-NEXT: s_add_i32 s6, s10, -1
+; VI-NEXT: s_sub_i32 s7, s7, s6
+; VI-NEXT: v_div_scale_f32 v2, s[12:13], 1.0, v0, 1.0
+; VI-NEXT: s_cmp_lg_u64 s[12:13], 0
+; VI-NEXT: v_rcp_f32_e32 v3, v1
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; VI-NEXT: v_fma_f32 v13, -v11, v12, 1.0
-; VI-NEXT: v_fma_f32 v12, v13, v12, v12
-; VI-NEXT: v_mul_f32_e32 v13, v8, v12
-; VI-NEXT: v_fma_f32 v14, -v11, v13, v8
-; VI-NEXT: v_fma_f32 v13, v14, v12, v13
-; VI-NEXT: v_fma_f32 v8, -v11, v13, v8
+; VI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; VI-NEXT: v_fma_f32 v3, v4, v3, v3
+; VI-NEXT: v_mul_f32_e32 v4, v2, v3
+; VI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; VI-NEXT: v_fma_f32 v4, v5, v3, v4
+; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; VI-NEXT: v_div_fmas_f32 v8, v8, v12, v13
-; VI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v6
-; VI-NEXT: v_div_fixup_f32 v8, v8, v5, 1.0
-; VI-NEXT: s_cbranch_vccnz .LBB10_8
-; VI-NEXT: ; %bb.5: ; %frem.loop_body93.preheader
-; VI-NEXT: v_sub_u32_e32 v6, vcc, v9, v10
-; VI-NEXT: v_add_u32_e32 v6, vcc, 11, v6
-; VI-NEXT: .LBB10_6: ; %frem.loop_body93
+; VI-NEXT: s_cselect_b64 vcc, exec, 0
+; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_cmp_le_i32 s7, 11
+; VI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; VI-NEXT: s_cbranch_scc1 .LBB10_6
+; VI-NEXT: ; %bb.4: ; %frem.loop_body93.preheader
+; VI-NEXT: s_add_i32 s9, s9, 11
+; VI-NEXT: s_sub_i32 s7, s9, s10
+; VI-NEXT: .LBB10_5: ; %frem.loop_body93
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v9, v7
-; VI-NEXT: v_mul_f32_e32 v7, v9, v8
-; VI-NEXT: v_rndne_f32_e32 v7, v7
-; VI-NEXT: v_fma_f32 v7, -v7, v5, v9
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v7
-; VI-NEXT: v_add_f32_e32 v10, v7, v5
-; VI-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
-; VI-NEXT: v_add_u32_e32 v6, vcc, -11, v6
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v6
-; VI-NEXT: v_ldexp_f32 v7, v7, 11
-; VI-NEXT: s_cbranch_vccnz .LBB10_6
-; VI-NEXT: ; %bb.7: ; %Flow133
-; VI-NEXT: v_mov_b32_e32 v7, v9
-; VI-NEXT: .LBB10_8: ; %frem.loop_exit94
-; VI-NEXT: v_add_u32_e32 v6, vcc, -10, v6
-; VI-NEXT: v_ldexp_f32 v6, v7, v6
-; VI-NEXT: v_mul_f32_e32 v7, v6, v8
-; VI-NEXT: v_rndne_f32_e32 v7, v7
-; VI-NEXT: v_fma_f32 v6, -v7, v5, v6
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v6
-; VI-NEXT: v_add_f32_e32 v5, v6, v5
-; VI-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
-; VI-NEXT: v_ldexp_f32 v4, v5, v4
-; VI-NEXT: v_cvt_f16_f32_e32 v4, v4
-; VI-NEXT: s_movk_i32 s2, 0x7fff
-; VI-NEXT: v_bfi_b32 v4, s2, v4, v0
-; VI-NEXT: .LBB10_9:
-; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; VI-NEXT: v_cvt_f32_f16_e64 v9, |v5|
-; VI-NEXT: v_cvt_f32_f16_e64 v8, |v6|
-; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v9, v8
-; VI-NEXT: s_cbranch_vccz .LBB10_11
-; VI-NEXT: ; %bb.10: ; %frem.else53
-; VI-NEXT: v_and_b32_e32 v7, 0x8000, v5
-; VI-NEXT: v_cmp_eq_f32_e32 vcc, v9, v8
-; VI-NEXT: v_cndmask_b32_e32 v7, v5, v7, vcc
-; VI-NEXT: s_mov_b64 s[2:3], 0
-; VI-NEXT: s_branch .LBB10_12
-; VI-NEXT: .LBB10_11:
-; VI-NEXT: s_mov_b64 s[2:3], -1
-; VI-NEXT: ; implicit-def: $vgpr7
-; VI-NEXT: .LBB10_12: ; %Flow131
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cbranch_scc1 .LBB10_18
-; VI-NEXT: ; %bb.13: ; %frem.compute52
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v12, v9
-; VI-NEXT: v_frexp_mant_f32_e32 v7, v9
-; VI-NEXT: v_frexp_mant_f32_e32 v9, v8
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v13, v8
-; VI-NEXT: v_ldexp_f32 v8, v9, 1
-; VI-NEXT: v_div_scale_f32 v14, s[2:3], v8, v8, 1.0
-; VI-NEXT: v_ldexp_f32 v10, v7, 11
-; VI-NEXT: v_add_u32_e32 v7, vcc, -1, v13
-; VI-NEXT: v_not_b32_e32 v9, v7
-; VI-NEXT: v_add_u32_e32 v9, vcc, v9, v12
-; VI-NEXT: v_div_scale_f32 v11, vcc, 1.0, v8, 1.0
-; VI-NEXT: v_rcp_f32_e32 v15, v14
+; VI-NEXT: v_mul_f32_e32 v2, s8, v1
+; VI-NEXT: v_rndne_f32_e32 v2, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s8
+; VI-NEXT: v_readfirstlane_b32 s9, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; VI-NEXT: v_add_f32_e32 v2, v2, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s10, v2
+; VI-NEXT: s_cselect_b32 s9, s10, s9
+; VI-NEXT: v_ldexp_f32 v2, s9, 11
+; VI-NEXT: v_readfirstlane_b32 s10, v2
+; VI-NEXT: s_add_i32 s7, s7, -11
+; VI-NEXT: s_cmp_gt_i32 s7, 11
+; VI-NEXT: s_mov_b32 s9, s8
+; VI-NEXT: s_mov_b32 s8, s10
+; VI-NEXT: s_cbranch_scc1 .LBB10_5
+; VI-NEXT: s_branch .LBB10_7
+; VI-NEXT: .LBB10_6:
+; VI-NEXT: s_mov_b32 s9, s8
+; VI-NEXT: .LBB10_7: ; %frem.loop_exit94
+; VI-NEXT: s_add_i32 s7, s7, -10
+; VI-NEXT: v_mov_b32_e32 v2, s7
+; VI-NEXT: v_ldexp_f32 v2, s9, v2
+; VI-NEXT: v_mul_f32_e32 v1, v2, v1
+; VI-NEXT: v_rndne_f32_e32 v1, v1
+; VI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: v_readfirstlane_b32 s7, v1
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s8, v0
+; VI-NEXT: s_cselect_b32 s7, s8, s7
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: v_ldexp_f32 v0, s7, v0
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: s_and_b32 s7, s2, 0xffff8000
+; VI-NEXT: v_readfirstlane_b32 s6, v0
+; VI-NEXT: s_and_b32 s6, s6, 0x7fff
+; VI-NEXT: s_or_b32 s6, s6, s7
+; VI-NEXT: .LBB10_8: ; %Flow136
+; VI-NEXT: s_lshr_b32 s7, s2, 16
+; VI-NEXT: s_lshr_b32 s8, s4, 16
+; VI-NEXT: v_cvt_f32_f16_e64 v1, |s7|
+; VI-NEXT: v_cvt_f32_f16_e64 v0, |s8|
+; VI-NEXT: s_mov_b32 s10, 1
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_cbranch_scc0 .LBB10_10
+; VI-NEXT: ; %bb.9: ; %frem.else53
+; VI-NEXT: s_and_b32 s9, s7, 0xffff8000
+; VI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_mov_b32 s10, 0
+; VI-NEXT: s_cselect_b32 s9, s9, s7
+; VI-NEXT: .LBB10_10: ; %Flow131
+; VI-NEXT: s_xor_b32 s10, s10, 1
+; VI-NEXT: s_cmp_lg_u32 s10, 0
+; VI-NEXT: s_cbranch_scc1 .LBB10_16
+; VI-NEXT: ; %bb.11: ; %frem.compute52
+; VI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; VI-NEXT: v_readfirstlane_b32 s12, v1
+; VI-NEXT: v_ldexp_f32 v1, v2, 11
+; VI-NEXT: v_readfirstlane_b32 s11, v1
+; VI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; VI-NEXT: v_readfirstlane_b32 s13, v0
+; VI-NEXT: v_ldexp_f32 v0, v1, 1
+; VI-NEXT: v_div_scale_f32 v1, s[14:15], v0, v0, 1.0
+; VI-NEXT: s_add_i32 s10, s12, -1
+; VI-NEXT: s_add_i32 s9, s13, -1
+; VI-NEXT: s_sub_i32 s10, s10, s9
+; VI-NEXT: v_div_scale_f32 v2, s[14:15], 1.0, v0, 1.0
+; VI-NEXT: s_cmp_lg_u64 s[14:15], 0
+; VI-NEXT: v_rcp_f32_e32 v3, v1
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; VI-NEXT: v_fma_f32 v16, -v14, v15, 1.0
-; VI-NEXT: v_fma_f32 v15, v16, v15, v15
-; VI-NEXT: v_mul_f32_e32 v16, v11, v15
-; VI-NEXT: v_fma_f32 v17, -v14, v16, v11
-; VI-NEXT: v_fma_f32 v16, v17, v15, v16
-; VI-NEXT: v_fma_f32 v11, -v14, v16, v11
+; VI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; VI-NEXT: v_fma_f32 v3, v4, v3, v3
+; VI-NEXT: v_mul_f32_e32 v4, v2, v3
+; VI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; VI-NEXT: v_fma_f32 v4, v5, v3, v4
+; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; VI-NEXT: v_div_fmas_f32 v11, v11, v15, v16
-; VI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v9
-; VI-NEXT: v_div_fixup_f32 v11, v11, v8, 1.0
-; VI-NEXT: s_cbranch_vccnz .LBB10_17
-; VI-NEXT: ; %bb.14: ; %frem.loop_body60.preheader
-; VI-NEXT: v_sub_u32_e32 v9, vcc, v12, v13
-; VI-NEXT: v_add_u32_e32 v9, vcc, 11, v9
-; VI-NEXT: .LBB10_15: ; %frem.loop_body60
+; VI-NEXT: s_cselect_b64 vcc, exec, 0
+; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_cmp_le_i32 s10, 11
+; VI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; VI-NEXT: s_cbranch_scc1 .LBB10_14
+; VI-NEXT: ; %bb.12: ; %frem.loop_body60.preheader
+; VI-NEXT: s_add_i32 s12, s12, 11
+; VI-NEXT: s_sub_i32 s10, s12, s13
+; VI-NEXT: .LBB10_13: ; %frem.loop_body60
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v12, v10
-; VI-NEXT: v_mul_f32_e32 v10, v12, v11
-; VI-NEXT: v_rndne_f32_e32 v10, v10
-; VI-NEXT: v_fma_f32 v10, -v10, v8, v12
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v10
-; VI-NEXT: v_add_f32_e32 v13, v10, v8
-; VI-NEXT: v_cndmask_b32_e32 v10, v10, v13, vcc
-; VI-NEXT: v_add_u32_e32 v9, vcc, -11, v9
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v9
-; VI-NEXT: v_ldexp_f32 v10, v10, 11
-; VI-NEXT: s_cbranch_vccnz .LBB10_15
-; VI-NEXT: ; %bb.16: ; %Flow129
-; VI-NEXT: v_mov_b32_e32 v10, v12
-; VI-NEXT: .LBB10_17: ; %frem.loop_exit61
-; VI-NEXT: v_add_u32_e32 v9, vcc, -10, v9
-; VI-NEXT: v_ldexp_f32 v9, v10, v9
-; VI-NEXT: v_mul_f32_e32 v10, v9, v11
-; VI-NEXT: v_rndne_f32_e32 v10, v10
-; VI-NEXT: v_fma_f32 v9, -v10, v8, v9
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v9
-; VI-NEXT: v_add_f32_e32 v8, v9, v8
-; VI-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
-; VI-NEXT: v_ldexp_f32 v7, v8, v7
-; VI-NEXT: v_cvt_f16_f32_e32 v7, v7
-; VI-NEXT: s_movk_i32 s2, 0x7fff
-; VI-NEXT: v_bfi_b32 v7, s2, v7, v5
-; VI-NEXT: .LBB10_18:
-; VI-NEXT: v_cvt_f32_f16_e64 v10, |v1|
-; VI-NEXT: v_cvt_f32_f16_e64 v9, |v3|
-; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v10, v9
-; VI-NEXT: s_cbranch_vccz .LBB10_20
-; VI-NEXT: ; %bb.19: ; %frem.else20
-; VI-NEXT: v_and_b32_e32 v8, 0x8000, v1
-; VI-NEXT: v_cmp_eq_f32_e32 vcc, v10, v9
-; VI-NEXT: v_cndmask_b32_e32 v8, v1, v8, vcc
-; VI-NEXT: s_mov_b64 s[2:3], 0
-; VI-NEXT: s_branch .LBB10_21
-; VI-NEXT: .LBB10_20:
-; VI-NEXT: s_mov_b64 s[2:3], -1
-; VI-NEXT: ; implicit-def: $vgpr8
-; VI-NEXT: .LBB10_21: ; %Flow127
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cbranch_scc1 .LBB10_27
-; VI-NEXT: ; %bb.22: ; %frem.compute19
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v13, v10
-; VI-NEXT: v_frexp_mant_f32_e32 v8, v10
-; VI-NEXT: v_frexp_mant_f32_e32 v10, v9
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v14, v9
-; VI-NEXT: v_ldexp_f32 v9, v10, 1
-; VI-NEXT: v_div_scale_f32 v15, s[2:3], v9, v9, 1.0
-; VI-NEXT: v_ldexp_f32 v11, v8, 11
-; VI-NEXT: v_add_u32_e32 v8, vcc, -1, v14
-; VI-NEXT: v_not_b32_e32 v10, v8
-; VI-NEXT: v_add_u32_e32 v10, vcc, v10, v13
-; VI-NEXT: v_div_scale_f32 v12, vcc, 1.0, v9, 1.0
-; VI-NEXT: v_rcp_f32_e32 v16, v15
+; VI-NEXT: v_mul_f32_e32 v2, s11, v1
+; VI-NEXT: v_rndne_f32_e32 v2, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s11
+; VI-NEXT: v_readfirstlane_b32 s12, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; VI-NEXT: v_add_f32_e32 v2, v2, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s13, v2
+; VI-NEXT: s_cselect_b32 s12, s13, s12
+; VI-NEXT: v_ldexp_f32 v2, s12, 11
+; VI-NEXT: v_readfirstlane_b32 s13, v2
+; VI-NEXT: s_add_i32 s10, s10, -11
+; VI-NEXT: s_cmp_gt_i32 s10, 11
+; VI-NEXT: s_mov_b32 s12, s11
+; VI-NEXT: s_mov_b32 s11, s13
+; VI-NEXT: s_cbranch_scc1 .LBB10_13
+; VI-NEXT: s_branch .LBB10_15
+; VI-NEXT: .LBB10_14:
+; VI-NEXT: s_mov_b32 s12, s11
+; VI-NEXT: .LBB10_15: ; %frem.loop_exit61
+; VI-NEXT: s_add_i32 s10, s10, -10
+; VI-NEXT: v_mov_b32_e32 v2, s10
+; VI-NEXT: v_ldexp_f32 v2, s12, v2
+; VI-NEXT: v_mul_f32_e32 v1, v2, v1
+; VI-NEXT: v_rndne_f32_e32 v1, v1
+; VI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: v_readfirstlane_b32 s10, v1
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s11, v0
+; VI-NEXT: s_cselect_b32 s10, s11, s10
+; VI-NEXT: v_mov_b32_e32 v0, s9
+; VI-NEXT: v_ldexp_f32 v0, s10, v0
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: s_and_b32 s10, s7, 0xffff8000
+; VI-NEXT: v_readfirstlane_b32 s9, v0
+; VI-NEXT: s_and_b32 s9, s9, 0x7fff
+; VI-NEXT: s_or_b32 s9, s9, s10
+; VI-NEXT: .LBB10_16: ; %Flow132
+; VI-NEXT: v_cvt_f32_f16_e64 v1, |s3|
+; VI-NEXT: v_cvt_f32_f16_e64 v0, |s5|
+; VI-NEXT: s_mov_b32 s11, 1
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_cbranch_scc0 .LBB10_18
+; VI-NEXT: ; %bb.17: ; %frem.else20
+; VI-NEXT: s_and_b32 s10, s3, 0xffff8000
+; VI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_mov_b32 s11, 0
+; VI-NEXT: s_cselect_b32 s10, s10, s3
+; VI-NEXT: .LBB10_18: ; %Flow127
+; VI-NEXT: s_xor_b32 s11, s11, 1
+; VI-NEXT: s_cmp_lg_u32 s11, 0
+; VI-NEXT: s_cbranch_scc1 .LBB10_24
+; VI-NEXT: ; %bb.19: ; %frem.compute19
+; VI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; VI-NEXT: v_readfirstlane_b32 s13, v1
+; VI-NEXT: v_ldexp_f32 v1, v2, 11
+; VI-NEXT: v_readfirstlane_b32 s12, v1
+; VI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; VI-NEXT: v_readfirstlane_b32 s14, v0
+; VI-NEXT: v_ldexp_f32 v0, v1, 1
+; VI-NEXT: v_div_scale_f32 v1, s[16:17], v0, v0, 1.0
+; VI-NEXT: s_add_i32 s11, s13, -1
+; VI-NEXT: s_add_i32 s10, s14, -1
+; VI-NEXT: s_sub_i32 s11, s11, s10
+; VI-NEXT: v_div_scale_f32 v2, s[16:17], 1.0, v0, 1.0
+; VI-NEXT: s_cmp_lg_u64 s[16:17], 0
+; VI-NEXT: v_rcp_f32_e32 v3, v1
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; VI-NEXT: v_fma_f32 v17, -v15, v16, 1.0
-; VI-NEXT: v_fma_f32 v16, v17, v16, v16
-; VI-NEXT: v_mul_f32_e32 v17, v12, v16
-; VI-NEXT: v_fma_f32 v18, -v15, v17, v12
-; VI-NEXT: v_fma_f32 v17, v18, v16, v17
-; VI-NEXT: v_fma_f32 v12, -v15, v17, v12
+; VI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; VI-NEXT: v_fma_f32 v3, v4, v3, v3
+; VI-NEXT: v_mul_f32_e32 v4, v2, v3
+; VI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; VI-NEXT: v_fma_f32 v4, v5, v3, v4
+; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; VI-NEXT: v_div_fmas_f32 v12, v12, v16, v17
-; VI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v10
-; VI-NEXT: v_div_fixup_f32 v12, v12, v9, 1.0
-; VI-NEXT: s_cbranch_vccnz .LBB10_26
-; VI-NEXT: ; %bb.23: ; %frem.loop_body27.preheader
-; VI-NEXT: v_sub_u32_e32 v10, vcc, v13, v14
-; VI-NEXT: v_add_u32_e32 v10, vcc, 11, v10
-; VI-NEXT: .LBB10_24: ; %frem.loop_body27
+; VI-NEXT: s_cselect_b64 vcc, exec, 0
+; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_cmp_le_i32 s11, 11
+; VI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; VI-NEXT: s_cbranch_scc1 .LBB10_22
+; VI-NEXT: ; %bb.20: ; %frem.loop_body27.preheader
+; VI-NEXT: s_add_i32 s13, s13, 11
+; VI-NEXT: s_sub_i32 s11, s13, s14
+; VI-NEXT: .LBB10_21: ; %frem.loop_body27
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v13, v11
-; VI-NEXT: v_mul_f32_e32 v11, v13, v12
-; VI-NEXT: v_rndne_f32_e32 v11, v11
-; VI-NEXT: v_fma_f32 v11, -v11, v9, v13
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v11
-; VI-NEXT: v_add_f32_e32 v14, v11, v9
-; VI-NEXT: v_cndmask_b32_e32 v11, v11, v14, vcc
-; VI-NEXT: v_add_u32_e32 v10, vcc, -11, v10
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v10
-; VI-NEXT: v_ldexp_f32 v11, v11, 11
-; VI-NEXT: s_cbranch_vccnz .LBB10_24
-; VI-NEXT: ; %bb.25: ; %Flow125
-; VI-NEXT: v_mov_b32_e32 v11, v13
-; VI-NEXT: .LBB10_26: ; %frem.loop_exit28
-; VI-NEXT: v_add_u32_e32 v10, vcc, -10, v10
-; VI-NEXT: v_ldexp_f32 v10, v11, v10
-; VI-NEXT: v_mul_f32_e32 v11, v10, v12
-; VI-NEXT: v_rndne_f32_e32 v11, v11
-; VI-NEXT: v_fma_f32 v10, -v11, v9, v10
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v10
-; VI-NEXT: v_add_f32_e32 v9, v10, v9
-; VI-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc
-; VI-NEXT: v_ldexp_f32 v8, v9, v8
-; VI-NEXT: v_cvt_f16_f32_e32 v8, v8
-; VI-NEXT: s_movk_i32 s2, 0x7fff
-; VI-NEXT: v_bfi_b32 v8, s2, v8, v1
-; VI-NEXT: .LBB10_27:
-; VI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; VI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
-; VI-NEXT: v_cvt_f32_f16_e64 v13, |v9|
-; VI-NEXT: v_cvt_f32_f16_e64 v12, |v10|
-; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v13, v12
-; VI-NEXT: s_cbranch_vccz .LBB10_29
-; VI-NEXT: ; %bb.28: ; %frem.else
-; VI-NEXT: v_and_b32_e32 v11, 0x8000, v9
-; VI-NEXT: v_cmp_eq_f32_e32 vcc, v13, v12
-; VI-NEXT: v_cndmask_b32_e32 v11, v9, v11, vcc
-; VI-NEXT: s_mov_b64 s[2:3], 0
-; VI-NEXT: s_branch .LBB10_30
-; VI-NEXT: .LBB10_29:
-; VI-NEXT: s_mov_b64 s[2:3], -1
-; VI-NEXT: ; implicit-def: $vgpr11
-; VI-NEXT: .LBB10_30: ; %Flow123
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cbranch_scc1 .LBB10_36
-; VI-NEXT: ; %bb.31: ; %frem.compute
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v16, v13
-; VI-NEXT: v_frexp_mant_f32_e32 v11, v13
-; VI-NEXT: v_frexp_mant_f32_e32 v13, v12
-; VI-NEXT: v_frexp_exp_i32_f32_e32 v17, v12
-; VI-NEXT: v_ldexp_f32 v12, v13, 1
-; VI-NEXT: v_div_scale_f32 v18, s[2:3], v12, v12, 1.0
-; VI-NEXT: v_ldexp_f32 v14, v11, 11
-; VI-NEXT: v_add_u32_e32 v11, vcc, -1, v17
-; VI-NEXT: v_not_b32_e32 v13, v11
-; VI-NEXT: v_add_u32_e32 v13, vcc, v13, v16
-; VI-NEXT: v_div_scale_f32 v15, vcc, 1.0, v12, 1.0
-; VI-NEXT: v_rcp_f32_e32 v19, v18
+; VI-NEXT: v_mul_f32_e32 v2, s12, v1
+; VI-NEXT: v_rndne_f32_e32 v2, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s12
+; VI-NEXT: v_readfirstlane_b32 s13, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; VI-NEXT: v_add_f32_e32 v2, v2, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s14, v2
+; VI-NEXT: s_cselect_b32 s13, s14, s13
+; VI-NEXT: v_ldexp_f32 v2, s13, 11
+; VI-NEXT: v_readfirstlane_b32 s14, v2
+; VI-NEXT: s_add_i32 s11, s11, -11
+; VI-NEXT: s_cmp_gt_i32 s11, 11
+; VI-NEXT: s_mov_b32 s13, s12
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_cbranch_scc1 .LBB10_21
+; VI-NEXT: s_branch .LBB10_23
+; VI-NEXT: .LBB10_22:
+; VI-NEXT: s_mov_b32 s13, s12
+; VI-NEXT: .LBB10_23: ; %frem.loop_exit28
+; VI-NEXT: s_add_i32 s11, s11, -10
+; VI-NEXT: v_mov_b32_e32 v2, s11
+; VI-NEXT: v_ldexp_f32 v2, s13, v2
+; VI-NEXT: v_mul_f32_e32 v1, v2, v1
+; VI-NEXT: v_rndne_f32_e32 v1, v1
+; VI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: v_readfirstlane_b32 s11, v1
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s12, v0
+; VI-NEXT: s_cselect_b32 s11, s12, s11
+; VI-NEXT: v_mov_b32_e32 v0, s10
+; VI-NEXT: v_ldexp_f32 v0, s11, v0
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: s_and_b32 s11, s3, 0xffff8000
+; VI-NEXT: v_readfirstlane_b32 s10, v0
+; VI-NEXT: s_and_b32 s10, s10, 0x7fff
+; VI-NEXT: s_or_b32 s10, s10, s11
+; VI-NEXT: .LBB10_24: ; %Flow128
+; VI-NEXT: s_lshr_b32 s11, s3, 16
+; VI-NEXT: s_lshr_b32 s12, s5, 16
+; VI-NEXT: v_cvt_f32_f16_e64 v1, |s11|
+; VI-NEXT: v_cvt_f32_f16_e64 v0, |s12|
+; VI-NEXT: s_mov_b32 s14, 1
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: v_cmp_ngt_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_cbranch_scc0 .LBB10_26
+; VI-NEXT: ; %bb.25: ; %frem.else
+; VI-NEXT: s_and_b32 s13, s11, 0xffff8000
+; VI-NEXT: v_cmp_eq_f32_e32 vcc, v1, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: s_mov_b32 s14, 0
+; VI-NEXT: s_cselect_b32 s13, s13, s11
+; VI-NEXT: .LBB10_26: ; %Flow123
+; VI-NEXT: s_xor_b32 s14, s14, 1
+; VI-NEXT: s_cmp_lg_u32 s14, 0
+; VI-NEXT: s_cbranch_scc1 .LBB10_32
+; VI-NEXT: ; %bb.27: ; %frem.compute
+; VI-NEXT: v_frexp_mant_f32_e32 v2, v1
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; VI-NEXT: v_readfirstlane_b32 s16, v1
+; VI-NEXT: v_ldexp_f32 v1, v2, 11
+; VI-NEXT: v_readfirstlane_b32 s15, v1
+; VI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; VI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; VI-NEXT: v_readfirstlane_b32 s17, v0
+; VI-NEXT: v_ldexp_f32 v0, v1, 1
+; VI-NEXT: v_div_scale_f32 v1, s[18:19], v0, v0, 1.0
+; VI-NEXT: s_add_i32 s14, s16, -1
+; VI-NEXT: s_add_i32 s13, s17, -1
+; VI-NEXT: s_sub_i32 s14, s14, s13
+; VI-NEXT: v_div_scale_f32 v2, s[18:19], 1.0, v0, 1.0
+; VI-NEXT: s_cmp_lg_u64 s[18:19], 0
+; VI-NEXT: v_rcp_f32_e32 v3, v1
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; VI-NEXT: v_fma_f32 v20, -v18, v19, 1.0
-; VI-NEXT: v_fma_f32 v19, v20, v19, v19
-; VI-NEXT: v_mul_f32_e32 v20, v15, v19
-; VI-NEXT: v_fma_f32 v21, -v18, v20, v15
-; VI-NEXT: v_fma_f32 v20, v21, v19, v20
-; VI-NEXT: v_fma_f32 v15, -v18, v20, v15
+; VI-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; VI-NEXT: v_fma_f32 v3, v4, v3, v3
+; VI-NEXT: v_mul_f32_e32 v4, v2, v3
+; VI-NEXT: v_fma_f32 v5, -v1, v4, v2
+; VI-NEXT: v_fma_f32 v4, v5, v3, v4
+; VI-NEXT: v_fma_f32 v1, -v1, v4, v2
; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; VI-NEXT: v_div_fmas_f32 v15, v15, v19, v20
-; VI-NEXT: v_cmp_gt_i32_e32 vcc, 12, v13
-; VI-NEXT: v_div_fixup_f32 v15, v15, v12, 1.0
-; VI-NEXT: s_cbranch_vccnz .LBB10_35
-; VI-NEXT: ; %bb.32: ; %frem.loop_body.preheader
-; VI-NEXT: v_sub_u32_e32 v13, vcc, v16, v17
-; VI-NEXT: v_add_u32_e32 v13, vcc, 11, v13
-; VI-NEXT: .LBB10_33: ; %frem.loop_body
+; VI-NEXT: s_cselect_b64 vcc, exec, 0
+; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4
+; VI-NEXT: s_cmp_le_i32 s14, 11
+; VI-NEXT: v_div_fixup_f32 v1, v1, v0, 1.0
+; VI-NEXT: s_cbranch_scc1 .LBB10_30
+; VI-NEXT: ; %bb.28: ; %frem.loop_body.preheader
+; VI-NEXT: s_add_i32 s16, s16, 11
+; VI-NEXT: s_sub_i32 s14, s16, s17
+; VI-NEXT: .LBB10_29: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v16, v14
-; VI-NEXT: v_mul_f32_e32 v14, v16, v15
-; VI-NEXT: v_rndne_f32_e32 v14, v14
-; VI-NEXT: v_fma_f32 v14, -v14, v12, v16
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v14
-; VI-NEXT: v_add_f32_e32 v17, v14, v12
-; VI-NEXT: v_cndmask_b32_e32 v14, v14, v17, vcc
-; VI-NEXT: v_add_u32_e32 v13, vcc, -11, v13
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, 11, v13
-; VI-NEXT: v_ldexp_f32 v14, v14, 11
-; VI-NEXT: s_cbranch_vccnz .LBB10_33
-; VI-NEXT: ; %bb.34: ; %Flow
-; VI-NEXT: v_mov_b32_e32 v14, v16
-; VI-NEXT: .LBB10_35: ; %frem.loop_exit
-; VI-NEXT: v_add_u32_e32 v13, vcc, -10, v13
-; VI-NEXT: v_ldexp_f32 v13, v14, v13
-; VI-NEXT: v_mul_f32_e32 v14, v13, v15
-; VI-NEXT: v_rndne_f32_e32 v14, v14
-; VI-NEXT: v_fma_f32 v13, -v14, v12, v13
-; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v13
-; VI-NEXT: v_add_f32_e32 v12, v13, v12
-; VI-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
-; VI-NEXT: v_ldexp_f32 v11, v12, v11
-; VI-NEXT: v_cvt_f16_f32_e32 v11, v11
-; VI-NEXT: s_movk_i32 s2, 0x7fff
-; VI-NEXT: v_bfi_b32 v11, s2, v11, v9
-; VI-NEXT: .LBB10_36: ; %Flow124
-; VI-NEXT: s_movk_i32 s4, 0x7c00
-; VI-NEXT: v_cmp_lg_f16_e32 vcc, 0, v2
-; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |v0|, s4
-; VI-NEXT: s_and_b64 vcc, s[2:3], vcc
-; VI-NEXT: v_mov_b32_e32 v2, 0x7e00
-; VI-NEXT: v_cndmask_b32_e32 v4, v2, v4, vcc
-; VI-NEXT: v_cmp_lg_f16_e32 vcc, 0, v6
-; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |v5|, s4
-; VI-NEXT: s_and_b64 vcc, s[2:3], vcc
-; VI-NEXT: v_cndmask_b32_sdwa v5, v2, v7, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_cmp_lg_f16_e32 vcc, 0, v3
-; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |v1|, s4
-; VI-NEXT: s_and_b64 vcc, s[2:3], vcc
-; VI-NEXT: v_cndmask_b32_e32 v3, v2, v8, vcc
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_cmp_lg_f16_e32 vcc, 0, v10
-; VI-NEXT: v_cmp_nge_f16_e64 s[0:1], |v9|, s4
-; VI-NEXT: s_and_b64 vcc, s[0:1], vcc
-; VI-NEXT: v_cndmask_b32_sdwa v2, v2, v11, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v2, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: v_mul_f32_e32 v2, s15, v1
+; VI-NEXT: v_rndne_f32_e32 v2, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s15
+; VI-NEXT: v_readfirstlane_b32 s16, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; VI-NEXT: v_add_f32_e32 v2, v2, v0
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s17, v2
+; VI-NEXT: s_cselect_b32 s16, s17, s16
+; VI-NEXT: v_ldexp_f32 v2, s16, 11
+; VI-NEXT: v_readfirstlane_b32 s17, v2
+; VI-NEXT: s_add_i32 s14, s14, -11
+; VI-NEXT: s_cmp_gt_i32 s14, 11
+; VI-NEXT: s_mov_b32 s16, s15
+; VI-NEXT: s_mov_b32 s15, s17
+; VI-NEXT: s_cbranch_scc1 .LBB10_29
+; VI-NEXT: s_branch .LBB10_31
+; VI-NEXT: .LBB10_30:
+; VI-NEXT: s_mov_b32 s16, s15
+; VI-NEXT: .LBB10_31: ; %frem.loop_exit
+; VI-NEXT: s_add_i32 s14, s14, -10
+; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_ldexp_f32 v2, s16, v2
+; VI-NEXT: v_mul_f32_e32 v1, v2, v1
+; VI-NEXT: v_rndne_f32_e32 v1, v1
+; VI-NEXT: v_fma_f32 v1, -v1, v0, v2
+; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v1
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: v_readfirstlane_b32 s14, v1
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-NEXT: v_readfirstlane_b32 s15, v0
+; VI-NEXT: s_cselect_b32 s14, s15, s14
+; VI-NEXT: v_mov_b32_e32 v0, s13
+; VI-NEXT: v_ldexp_f32 v0, s14, v0
+; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT: s_and_b32 s14, s11, 0xffff8000
+; VI-NEXT: v_readfirstlane_b32 s13, v0
+; VI-NEXT: s_and_b32 s13, s13, 0x7fff
+; VI-NEXT: s_or_b32 s13, s13, s14
+; VI-NEXT: .LBB10_32: ; %Flow124
+; VI-NEXT: v_cmp_nlg_f16_e64 s[14:15], s4, 0
+; VI-NEXT: v_mov_b32_e32 v0, 0x7c00
+; VI-NEXT: s_cmp_lg_u64 s[14:15], 0
+; VI-NEXT: v_cmp_nge_f16_e64 s[14:15], |s2|, v0
+; VI-NEXT: s_cselect_b32 s16, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[14:15], 0
+; VI-NEXT: v_cmp_nlg_f16_e64 s[14:15], s8, 0
+; VI-NEXT: s_cselect_b32 s18, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[14:15], 0
+; VI-NEXT: v_cmp_nge_f16_e64 s[14:15], |s7|, v0
+; VI-NEXT: s_cselect_b32 s8, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[14:15], 0
+; VI-NEXT: v_cmp_nlg_f16_e64 s[4:5], s5, 0
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
+; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |s3|, v0
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: v_cmp_nlg_f16_e64 s[2:3], s12, 0
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: v_cmp_nge_f16_e64 s[2:3], |s11|, v0
+; VI-NEXT: s_movk_i32 s17, 0x7e00
+; VI-NEXT: s_cselect_b32 s12, 1, 0
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: s_cselect_b32 s3, 1, 0
+; VI-NEXT: s_bfe_u32 s11, s17, 0x100000
+; VI-NEXT: s_bfe_u32 s2, s6, 0x100000
+; VI-NEXT: s_cmp_lg_u32 s16, 0
+; VI-NEXT: s_cselect_b32 s2, s11, s2
+; VI-NEXT: s_cmp_lg_u32 s18, 0
+; VI-NEXT: s_cselect_b32 s2, s2, s11
+; VI-NEXT: s_bfe_u32 s6, s9, 0x100000
+; VI-NEXT: s_cmp_lg_u32 s8, 0
+; VI-NEXT: s_cselect_b32 s6, s11, s6
+; VI-NEXT: s_cmp_lg_u32 s7, 0
+; VI-NEXT: s_cselect_b32 s6, s6, s11
+; VI-NEXT: s_lshl_b32 s6, s6, 16
+; VI-NEXT: s_or_b32 s2, s2, s6
+; VI-NEXT: s_bfe_u32 s6, s10, 0x100000
+; VI-NEXT: s_cmp_lg_u32 s4, 0
+; VI-NEXT: s_cselect_b32 s4, s11, s6
+; VI-NEXT: s_cmp_lg_u32 s5, 0
+; VI-NEXT: s_cselect_b32 s4, s4, s11
+; VI-NEXT: s_bfe_u32 s5, s13, 0x100000
+; VI-NEXT: s_cmp_lg_u32 s12, 0
+; VI-NEXT: s_cselect_b32 s5, s11, s5
+; VI-NEXT: s_cmp_lg_u32 s3, 0
+; VI-NEXT: s_cselect_b32 s3, s5, s11
+; VI-NEXT: s_lshl_b32 s3, s3, 16
+; VI-NEXT: s_or_b32 s3, s4, s3
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
%gep2 = getelementptr <4 x half>, ptr addrspace(1) %in2, i32 4
%r0 = load <4 x half>, ptr addrspace(1) %in1, align 16
@@ -2311,20 +2592,21 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s7, s9, s10
; CI-NEXT: .LBB11_5: ; %frem.loop_body23
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b32 s9, s8
-; CI-NEXT: v_mul_f32_e32 v2, s9, v1
+; CI-NEXT: v_mul_f32_e32 v2, s8, v1
; CI-NEXT: v_rndne_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v2, -v2, v0, s9
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s8
; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
-; CI-NEXT: v_readfirstlane_b32 s8, v2
+; CI-NEXT: v_readfirstlane_b32 s9, v2
; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
; CI-NEXT: v_add_f32_e32 v2, v2, v0
; CI-NEXT: v_readfirstlane_b32 s10, v2
-; CI-NEXT: s_cselect_b32 s8, s10, s8
-; CI-NEXT: v_ldexp_f32_e64 v2, s8, 12
+; CI-NEXT: s_cselect_b32 s9, s10, s9
+; CI-NEXT: v_ldexp_f32_e64 v2, s9, 12
+; CI-NEXT: v_readfirstlane_b32 s10, v2
; CI-NEXT: s_add_i32 s7, s7, -12
-; CI-NEXT: v_readfirstlane_b32 s8, v2
; CI-NEXT: s_cmp_gt_i32 s7, 12
+; CI-NEXT: s_mov_b32 s9, s8
+; CI-NEXT: s_mov_b32 s8, s10
; CI-NEXT: s_cbranch_scc1 .LBB11_5
; CI-NEXT: s_branch .LBB11_7
; CI-NEXT: .LBB11_6:
@@ -2401,20 +2683,21 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s8, s10, s11
; CI-NEXT: .LBB11_13: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b32 s10, s9
-; CI-NEXT: v_mul_f32_e32 v2, s10, v1
+; CI-NEXT: v_mul_f32_e32 v2, s9, v1
; CI-NEXT: v_rndne_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v2, -v2, v0, s10
-; CI-NEXT: v_readfirstlane_b32 s9, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s9
; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s12, v2
+; CI-NEXT: s_or_b64 s[10:11], vcc, vcc
; CI-NEXT: v_add_f32_e32 v2, v2, v0
-; CI-NEXT: s_or_b64 s[12:13], vcc, vcc
+; CI-NEXT: v_readfirstlane_b32 s10, v2
+; CI-NEXT: s_cselect_b32 s10, s10, s12
+; CI-NEXT: v_ldexp_f32_e64 v2, s10, 12
; CI-NEXT: v_readfirstlane_b32 s11, v2
-; CI-NEXT: s_cselect_b32 s9, s11, s9
-; CI-NEXT: v_ldexp_f32_e64 v2, s9, 12
; CI-NEXT: s_add_i32 s8, s8, -12
-; CI-NEXT: v_readfirstlane_b32 s9, v2
; CI-NEXT: s_cmp_gt_i32 s8, 12
+; CI-NEXT: s_mov_b32 s10, s9
+; CI-NEXT: s_mov_b32 s9, s11
; CI-NEXT: s_cbranch_scc1 .LBB11_13
; CI-NEXT: s_branch .LBB11_15
; CI-NEXT: .LBB11_14:
@@ -2519,20 +2802,21 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s7, s9, s10
; VI-NEXT: .LBB11_5: ; %frem.loop_body23
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b32 s9, s8
-; VI-NEXT: v_mul_f32_e32 v2, s9, v1
+; VI-NEXT: v_mul_f32_e32 v2, s8, v1
; VI-NEXT: v_rndne_f32_e32 v2, v2
-; VI-NEXT: v_fma_f32 v2, -v2, v0, s9
-; VI-NEXT: v_readfirstlane_b32 s8, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s8
+; VI-NEXT: v_readfirstlane_b32 s9, v2
; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
; VI-NEXT: v_add_f32_e32 v2, v2, v0
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s10, v2
-; VI-NEXT: s_cselect_b32 s8, s10, s8
-; VI-NEXT: v_ldexp_f32 v2, s8, 12
+; VI-NEXT: s_cselect_b32 s9, s10, s9
+; VI-NEXT: v_ldexp_f32 v2, s9, 12
+; VI-NEXT: v_readfirstlane_b32 s10, v2
; VI-NEXT: s_add_i32 s7, s7, -12
-; VI-NEXT: v_readfirstlane_b32 s8, v2
; VI-NEXT: s_cmp_gt_i32 s7, 12
+; VI-NEXT: s_mov_b32 s9, s8
+; VI-NEXT: s_mov_b32 s8, s10
; VI-NEXT: s_cbranch_scc1 .LBB11_5
; VI-NEXT: s_branch .LBB11_7
; VI-NEXT: .LBB11_6:
@@ -2609,20 +2893,21 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s8, s10, s11
; VI-NEXT: .LBB11_13: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b32 s10, s9
-; VI-NEXT: v_mul_f32_e32 v2, s10, v1
+; VI-NEXT: v_mul_f32_e32 v2, s9, v1
; VI-NEXT: v_rndne_f32_e32 v2, v2
-; VI-NEXT: v_fma_f32 v2, -v2, v0, s10
-; VI-NEXT: v_readfirstlane_b32 s9, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s9
+; VI-NEXT: v_readfirstlane_b32 s10, v2
; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
; VI-NEXT: v_add_f32_e32 v2, v2, v0
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s11, v2
-; VI-NEXT: s_cselect_b32 s9, s11, s9
-; VI-NEXT: v_ldexp_f32 v2, s9, 12
+; VI-NEXT: s_cselect_b32 s10, s11, s10
+; VI-NEXT: v_ldexp_f32 v2, s10, 12
+; VI-NEXT: v_readfirstlane_b32 s11, v2
; VI-NEXT: s_add_i32 s8, s8, -12
-; VI-NEXT: v_readfirstlane_b32 s9, v2
; VI-NEXT: s_cmp_gt_i32 s8, 12
+; VI-NEXT: s_mov_b32 s10, s9
+; VI-NEXT: s_mov_b32 s9, s11
; VI-NEXT: s_cbranch_scc1 .LBB11_13
; VI-NEXT: s_branch .LBB11_15
; VI-NEXT: .LBB11_14:
@@ -2735,20 +3020,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s3, s13, s14
; CI-NEXT: .LBB12_5: ; %frem.loop_body85
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b32 s13, s12
-; CI-NEXT: v_mul_f32_e32 v2, s13, v1
+; CI-NEXT: v_mul_f32_e32 v2, s12, v1
; CI-NEXT: v_rndne_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v2, -v2, v0, s13
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s12
; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
-; CI-NEXT: v_readfirstlane_b32 s12, v2
+; CI-NEXT: v_readfirstlane_b32 s13, v2
; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
; CI-NEXT: v_add_f32_e32 v2, v2, v0
; CI-NEXT: v_readfirstlane_b32 s14, v2
-; CI-NEXT: s_cselect_b32 s12, s14, s12
-; CI-NEXT: v_ldexp_f32_e64 v2, s12, 12
+; CI-NEXT: s_cselect_b32 s13, s14, s13
+; CI-NEXT: v_ldexp_f32_e64 v2, s13, 12
+; CI-NEXT: v_readfirstlane_b32 s14, v2
; CI-NEXT: s_add_i32 s3, s3, -12
-; CI-NEXT: v_readfirstlane_b32 s12, v2
; CI-NEXT: s_cmp_gt_i32 s3, 12
+; CI-NEXT: s_mov_b32 s13, s12
+; CI-NEXT: s_mov_b32 s12, s14
; CI-NEXT: s_cbranch_scc1 .LBB12_5
; CI-NEXT: s_branch .LBB12_7
; CI-NEXT: .LBB12_6:
@@ -2825,20 +3111,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s12, s14, s15
; CI-NEXT: .LBB12_13: ; %frem.loop_body54
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b32 s14, s13
-; CI-NEXT: v_mul_f32_e32 v2, s14, v1
+; CI-NEXT: v_mul_f32_e32 v2, s13, v1
; CI-NEXT: v_rndne_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v2, -v2, v0, s14
-; CI-NEXT: v_readfirstlane_b32 s13, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s13
; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s16, v2
+; CI-NEXT: s_or_b64 s[14:15], vcc, vcc
; CI-NEXT: v_add_f32_e32 v2, v2, v0
-; CI-NEXT: s_or_b64 s[16:17], vcc, vcc
+; CI-NEXT: v_readfirstlane_b32 s14, v2
+; CI-NEXT: s_cselect_b32 s14, s14, s16
+; CI-NEXT: v_ldexp_f32_e64 v2, s14, 12
; CI-NEXT: v_readfirstlane_b32 s15, v2
-; CI-NEXT: s_cselect_b32 s13, s15, s13
-; CI-NEXT: v_ldexp_f32_e64 v2, s13, 12
; CI-NEXT: s_add_i32 s12, s12, -12
-; CI-NEXT: v_readfirstlane_b32 s13, v2
; CI-NEXT: s_cmp_gt_i32 s12, 12
+; CI-NEXT: s_mov_b32 s14, s13
+; CI-NEXT: s_mov_b32 s13, s15
; CI-NEXT: s_cbranch_scc1 .LBB12_13
; CI-NEXT: s_branch .LBB12_15
; CI-NEXT: .LBB12_14:
@@ -2915,20 +3202,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s13, s15, s16
; CI-NEXT: .LBB12_21: ; %frem.loop_body23
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b32 s15, s14
-; CI-NEXT: v_mul_f32_e32 v2, s15, v1
+; CI-NEXT: v_mul_f32_e32 v2, s14, v1
; CI-NEXT: v_rndne_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v2, -v2, v0, s15
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s14
; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
-; CI-NEXT: v_readfirstlane_b32 s14, v2
+; CI-NEXT: v_readfirstlane_b32 s15, v2
; CI-NEXT: s_or_b64 s[16:17], vcc, vcc
; CI-NEXT: v_add_f32_e32 v2, v2, v0
; CI-NEXT: v_readfirstlane_b32 s16, v2
-; CI-NEXT: s_cselect_b32 s14, s16, s14
-; CI-NEXT: v_ldexp_f32_e64 v2, s14, 12
+; CI-NEXT: s_cselect_b32 s15, s16, s15
+; CI-NEXT: v_ldexp_f32_e64 v2, s15, 12
+; CI-NEXT: v_readfirstlane_b32 s16, v2
; CI-NEXT: s_add_i32 s13, s13, -12
-; CI-NEXT: v_readfirstlane_b32 s14, v2
; CI-NEXT: s_cmp_gt_i32 s13, 12
+; CI-NEXT: s_mov_b32 s15, s14
+; CI-NEXT: s_mov_b32 s14, s16
; CI-NEXT: s_cbranch_scc1 .LBB12_21
; CI-NEXT: s_branch .LBB12_23
; CI-NEXT: .LBB12_22:
@@ -3005,20 +3293,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s14, s16, s17
; CI-NEXT: .LBB12_29: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b32 s16, s15
-; CI-NEXT: v_mul_f32_e32 v2, s16, v1
+; CI-NEXT: v_mul_f32_e32 v2, s15, v1
; CI-NEXT: v_rndne_f32_e32 v2, v2
-; CI-NEXT: v_fma_f32 v2, -v2, v0, s16
-; CI-NEXT: v_readfirstlane_b32 s15, v2
+; CI-NEXT: v_fma_f32 v2, -v2, v0, s15
; CI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
+; CI-NEXT: v_readfirstlane_b32 s18, v2
+; CI-NEXT: s_or_b64 s[16:17], vcc, vcc
; CI-NEXT: v_add_f32_e32 v2, v2, v0
-; CI-NEXT: s_or_b64 s[18:19], vcc, vcc
+; CI-NEXT: v_readfirstlane_b32 s16, v2
+; CI-NEXT: s_cselect_b32 s16, s16, s18
+; CI-NEXT: v_ldexp_f32_e64 v2, s16, 12
; CI-NEXT: v_readfirstlane_b32 s17, v2
-; CI-NEXT: s_cselect_b32 s15, s17, s15
-; CI-NEXT: v_ldexp_f32_e64 v2, s15, 12
; CI-NEXT: s_add_i32 s14, s14, -12
-; CI-NEXT: v_readfirstlane_b32 s15, v2
; CI-NEXT: s_cmp_gt_i32 s14, 12
+; CI-NEXT: s_mov_b32 s16, s15
+; CI-NEXT: s_mov_b32 s15, s17
; CI-NEXT: s_cbranch_scc1 .LBB12_29
; CI-NEXT: s_branch .LBB12_31
; CI-NEXT: .LBB12_30:
@@ -3137,20 +3426,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s3, s13, s14
; VI-NEXT: .LBB12_5: ; %frem.loop_body85
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b32 s13, s12
-; VI-NEXT: v_mul_f32_e32 v2, s13, v1
+; VI-NEXT: v_mul_f32_e32 v2, s12, v1
; VI-NEXT: v_rndne_f32_e32 v2, v2
-; VI-NEXT: v_fma_f32 v2, -v2, v0, s13
-; VI-NEXT: v_readfirstlane_b32 s12, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s12
+; VI-NEXT: v_readfirstlane_b32 s13, v2
; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
; VI-NEXT: v_add_f32_e32 v2, v2, v0
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s14, v2
-; VI-NEXT: s_cselect_b32 s12, s14, s12
-; VI-NEXT: v_ldexp_f32 v2, s12, 12
+; VI-NEXT: s_cselect_b32 s13, s14, s13
+; VI-NEXT: v_ldexp_f32 v2, s13, 12
+; VI-NEXT: v_readfirstlane_b32 s14, v2
; VI-NEXT: s_add_i32 s3, s3, -12
-; VI-NEXT: v_readfirstlane_b32 s12, v2
; VI-NEXT: s_cmp_gt_i32 s3, 12
+; VI-NEXT: s_mov_b32 s13, s12
+; VI-NEXT: s_mov_b32 s12, s14
; VI-NEXT: s_cbranch_scc1 .LBB12_5
; VI-NEXT: s_branch .LBB12_7
; VI-NEXT: .LBB12_6:
@@ -3227,20 +3517,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s12, s14, s15
; VI-NEXT: .LBB12_13: ; %frem.loop_body54
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b32 s14, s13
-; VI-NEXT: v_mul_f32_e32 v2, s14, v1
+; VI-NEXT: v_mul_f32_e32 v2, s13, v1
; VI-NEXT: v_rndne_f32_e32 v2, v2
-; VI-NEXT: v_fma_f32 v2, -v2, v0, s14
-; VI-NEXT: v_readfirstlane_b32 s13, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s13
+; VI-NEXT: v_readfirstlane_b32 s14, v2
; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
; VI-NEXT: v_add_f32_e32 v2, v2, v0
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s15, v2
-; VI-NEXT: s_cselect_b32 s13, s15, s13
-; VI-NEXT: v_ldexp_f32 v2, s13, 12
+; VI-NEXT: s_cselect_b32 s14, s15, s14
+; VI-NEXT: v_ldexp_f32 v2, s14, 12
+; VI-NEXT: v_readfirstlane_b32 s15, v2
; VI-NEXT: s_add_i32 s12, s12, -12
-; VI-NEXT: v_readfirstlane_b32 s13, v2
; VI-NEXT: s_cmp_gt_i32 s12, 12
+; VI-NEXT: s_mov_b32 s14, s13
+; VI-NEXT: s_mov_b32 s13, s15
; VI-NEXT: s_cbranch_scc1 .LBB12_13
; VI-NEXT: s_branch .LBB12_15
; VI-NEXT: .LBB12_14:
@@ -3317,20 +3608,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s13, s15, s16
; VI-NEXT: .LBB12_21: ; %frem.loop_body23
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b32 s15, s14
-; VI-NEXT: v_mul_f32_e32 v2, s15, v1
+; VI-NEXT: v_mul_f32_e32 v2, s14, v1
; VI-NEXT: v_rndne_f32_e32 v2, v2
-; VI-NEXT: v_fma_f32 v2, -v2, v0, s15
-; VI-NEXT: v_readfirstlane_b32 s14, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s14
+; VI-NEXT: v_readfirstlane_b32 s15, v2
; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
; VI-NEXT: v_add_f32_e32 v2, v2, v0
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s16, v2
-; VI-NEXT: s_cselect_b32 s14, s16, s14
-; VI-NEXT: v_ldexp_f32 v2, s14, 12
+; VI-NEXT: s_cselect_b32 s15, s16, s15
+; VI-NEXT: v_ldexp_f32 v2, s15, 12
+; VI-NEXT: v_readfirstlane_b32 s16, v2
; VI-NEXT: s_add_i32 s13, s13, -12
-; VI-NEXT: v_readfirstlane_b32 s14, v2
; VI-NEXT: s_cmp_gt_i32 s13, 12
+; VI-NEXT: s_mov_b32 s15, s14
+; VI-NEXT: s_mov_b32 s14, s16
; VI-NEXT: s_cbranch_scc1 .LBB12_21
; VI-NEXT: s_branch .LBB12_23
; VI-NEXT: .LBB12_22:
@@ -3407,20 +3699,21 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s14, s16, s17
; VI-NEXT: .LBB12_29: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b32 s16, s15
-; VI-NEXT: v_mul_f32_e32 v2, s16, v1
+; VI-NEXT: v_mul_f32_e32 v2, s15, v1
; VI-NEXT: v_rndne_f32_e32 v2, v2
-; VI-NEXT: v_fma_f32 v2, -v2, v0, s16
-; VI-NEXT: v_readfirstlane_b32 s15, v2
+; VI-NEXT: v_fma_f32 v2, -v2, v0, s15
+; VI-NEXT: v_readfirstlane_b32 s16, v2
; VI-NEXT: v_cmp_gt_f32_e32 vcc, 0, v2
; VI-NEXT: v_add_f32_e32 v2, v2, v0
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s17, v2
-; VI-NEXT: s_cselect_b32 s15, s17, s15
-; VI-NEXT: v_ldexp_f32 v2, s15, 12
+; VI-NEXT: s_cselect_b32 s16, s17, s16
+; VI-NEXT: v_ldexp_f32 v2, s16, 12
+; VI-NEXT: v_readfirstlane_b32 s17, v2
; VI-NEXT: s_add_i32 s14, s14, -12
-; VI-NEXT: v_readfirstlane_b32 s15, v2
; VI-NEXT: s_cmp_gt_i32 s14, 12
+; VI-NEXT: s_mov_b32 s16, s15
+; VI-NEXT: s_mov_b32 s15, s17
; VI-NEXT: s_cbranch_scc1 .LBB12_29
; VI-NEXT: s_branch .LBB12_31
; VI-NEXT: .LBB12_30:
@@ -3552,23 +3845,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s15, s12, s13
; CI-NEXT: .LBB13_5: ; %frem.loop_body23
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b64 s[12:13], s[2:3]
-; CI-NEXT: v_mul_f64 v[4:5], s[12:13], v[2:3]
+; CI-NEXT: v_mul_f64 v[4:5], s[2:3], v[2:3]
; CI-NEXT: v_rndne_f64_e32 v[4:5], v[4:5]
-; CI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[12:13]
+; CI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[2:3]
; CI-NEXT: v_cmp_gt_f64_e32 vcc, 0, v[4:5]
-; CI-NEXT: v_readfirstlane_b32 s2, v4
-; CI-NEXT: v_readfirstlane_b32 s3, v5
+; CI-NEXT: v_readfirstlane_b32 s12, v4
+; CI-NEXT: v_readfirstlane_b32 s13, v5
; CI-NEXT: v_add_f64 v[4:5], v[4:5], v[0:1]
; CI-NEXT: s_or_b64 s[16:17], vcc, vcc
; CI-NEXT: v_readfirstlane_b32 s16, v4
; CI-NEXT: v_readfirstlane_b32 s17, v5
-; CI-NEXT: s_cselect_b64 s[2:3], s[16:17], s[2:3]
-; CI-NEXT: v_ldexp_f64 v[4:5], s[2:3], 26
+; CI-NEXT: s_cselect_b64 s[12:13], s[16:17], s[12:13]
+; CI-NEXT: v_ldexp_f64 v[4:5], s[12:13], 26
; CI-NEXT: s_sub_i32 s15, s15, 26
; CI-NEXT: s_cmp_gt_i32 s15, 26
-; CI-NEXT: v_readfirstlane_b32 s2, v4
-; CI-NEXT: v_readfirstlane_b32 s3, v5
+; CI-NEXT: s_mov_b64 s[12:13], s[2:3]
+; CI-NEXT: v_readfirstlane_b32 s16, v4
+; CI-NEXT: v_readfirstlane_b32 s17, v5
+; CI-NEXT: s_mov_b64 s[2:3], s[16:17]
; CI-NEXT: s_cbranch_scc1 .LBB13_5
; CI-NEXT: s_branch .LBB13_7
; CI-NEXT: .LBB13_6:
@@ -3655,23 +3949,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; CI-NEXT: s_sub_i32 s17, s14, s15
; CI-NEXT: .LBB13_13: ; %frem.loop_body
; CI-NEXT: ; =>This Inner Loop Header: Depth=1
-; CI-NEXT: s_mov_b64 s[14:15], s[12:13]
-; CI-NEXT: v_mul_f64 v[4:5], s[14:15], v[2:3]
+; CI-NEXT: v_mul_f64 v[4:5], s[12:13], v[2:3]
; CI-NEXT: v_rndne_f64_e32 v[4:5], v[4:5]
-; CI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[14:15]
+; CI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[12:13]
; CI-NEXT: v_cmp_gt_f64_e32 vcc, 0, v[4:5]
-; CI-NEXT: v_readfirstlane_b32 s12, v4
-; CI-NEXT: v_readfirstlane_b32 s13, v5
+; CI-NEXT: v_readfirstlane_b32 s14, v4
+; CI-NEXT: v_readfirstlane_b32 s15, v5
; CI-NEXT: v_add_f64 v[4:5], v[4:5], v[0:1]
; CI-NEXT: s_or_b64 s[18:19], vcc, vcc
; CI-NEXT: v_readfirstlane_b32 s18, v4
; CI-NEXT: v_readfirstlane_b32 s19, v5
-; CI-NEXT: s_cselect_b64 s[12:13], s[18:19], s[12:13]
-; CI-NEXT: v_ldexp_f64 v[4:5], s[12:13], 26
+; CI-NEXT: s_cselect_b64 s[14:15], s[18:19], s[14:15]
+; CI-NEXT: v_ldexp_f64 v[4:5], s[14:15], 26
; CI-NEXT: s_sub_i32 s17, s17, 26
; CI-NEXT: s_cmp_gt_i32 s17, 26
-; CI-NEXT: v_readfirstlane_b32 s12, v4
-; CI-NEXT: v_readfirstlane_b32 s13, v5
+; CI-NEXT: s_mov_b64 s[14:15], s[12:13]
+; CI-NEXT: v_readfirstlane_b32 s18, v4
+; CI-NEXT: v_readfirstlane_b32 s19, v5
+; CI-NEXT: s_mov_b64 s[12:13], s[18:19]
; CI-NEXT: s_cbranch_scc1 .LBB13_13
; CI-NEXT: s_branch .LBB13_15
; CI-NEXT: .LBB13_14:
@@ -3791,23 +4086,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s15, s12, s13
; VI-NEXT: .LBB13_5: ; %frem.loop_body23
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b64 s[12:13], s[2:3]
-; VI-NEXT: v_mul_f64 v[4:5], s[12:13], v[2:3]
+; VI-NEXT: v_mul_f64 v[4:5], s[2:3], v[2:3]
; VI-NEXT: v_rndne_f64_e32 v[4:5], v[4:5]
-; VI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[12:13]
-; VI-NEXT: v_readfirstlane_b32 s2, v4
-; VI-NEXT: v_readfirstlane_b32 s3, v5
+; VI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[2:3]
+; VI-NEXT: v_readfirstlane_b32 s12, v4
+; VI-NEXT: v_readfirstlane_b32 s13, v5
; VI-NEXT: v_cmp_gt_f64_e32 vcc, 0, v[4:5]
; VI-NEXT: v_add_f64 v[4:5], v[4:5], v[0:1]
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s16, v4
; VI-NEXT: v_readfirstlane_b32 s17, v5
-; VI-NEXT: s_cselect_b64 s[2:3], s[16:17], s[2:3]
-; VI-NEXT: v_ldexp_f64 v[4:5], s[2:3], 26
+; VI-NEXT: s_cselect_b64 s[12:13], s[16:17], s[12:13]
+; VI-NEXT: v_ldexp_f64 v[4:5], s[12:13], 26
; VI-NEXT: s_sub_i32 s15, s15, 26
; VI-NEXT: s_cmp_gt_i32 s15, 26
-; VI-NEXT: v_readfirstlane_b32 s2, v4
-; VI-NEXT: v_readfirstlane_b32 s3, v5
+; VI-NEXT: s_mov_b64 s[12:13], s[2:3]
+; VI-NEXT: v_readfirstlane_b32 s16, v4
+; VI-NEXT: v_readfirstlane_b32 s17, v5
+; VI-NEXT: s_mov_b64 s[2:3], s[16:17]
; VI-NEXT: s_cbranch_scc1 .LBB13_5
; VI-NEXT: s_branch .LBB13_7
; VI-NEXT: .LBB13_6:
@@ -3894,23 +4190,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_sub_i32 s17, s14, s15
; VI-NEXT: .LBB13_13: ; %frem.loop_body
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_mov_b64 s[14:15], s[12:13]
-; VI-NEXT: v_mul_f64 v[4:5], s[14:15], v[2:3]
+; VI-NEXT: v_mul_f64 v[4:5], s[12:13], v[2:3]
; VI-NEXT: v_rndne_f64_e32 v[4:5], v[4:5]
-; VI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[14:15]
-; VI-NEXT: v_readfirstlane_b32 s12, v4
-; VI-NEXT: v_readfirstlane_b32 s13, v5
+; VI-NEXT: v_fma_f64 v[4:5], -v[4:5], v[0:1], s[12:13]
+; VI-NEXT: v_readfirstlane_b32 s14, v4
+; VI-NEXT: v_readfirstlane_b32 s15, v5
; VI-NEXT: v_cmp_gt_f64_e32 vcc, 0, v[4:5]
; VI-NEXT: v_add_f64 v[4:5], v[4:5], v[0:1]
; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_readfirstlane_b32 s18, v4
; VI-NEXT: v_readfirstlane_b32 s19, v5
-; VI-NEXT: s_cselect_b64 s[12:13], s[18:19], s[12:13]
-; VI-NEXT: v_ldexp_f64 v[4:5], s[12:13], 26
+; VI-NEXT: s_cselect_b64 s[14:15], s[18:19], s[14:15]
+; VI-NEXT: v_ldexp_f64 v[4:5], s[14:15], 26
; VI-NEXT: s_sub_i32 s17, s17, 26
; VI-NEXT: s_cmp_gt_i32 s17, 26
-; VI-NEXT: v_readfirstlane_b32 s12, v4
-; VI-NEXT: v_readfirstlane_b32 s13, v5
+; VI-NEXT: s_mov_b64 s[14:15], s[12:13]
+; VI-NEXT: v_readfirstlane_b32 s18, v4
+; VI-NEXT: v_readfirstlane_b32 s19, v5
+; VI-NEXT: s_mov_b64 s[12:13], s[18:19]
; VI-NEXT: s_cbranch_scc1 .LBB13_13
; VI-NEXT: s_branch .LBB13_15
; VI-NEXT: .LBB13_14:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 63ae1c3af42be..0fea02b5fe904 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX6-LABEL: s_fshl_i7:
@@ -79,6 +79,47 @@ define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i7:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, -7
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_i32 s3, s3, -7
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
+; GFX11-NEXT: s_sub_i32 s3, 6, s2
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -113,7 +154,6 @@ define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -148,48 +188,6 @@ define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i7:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, -7
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_i32 s3, s3, -7
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX11-NEXT: s_sub_i32 s3, 6, s2
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
}
@@ -242,75 +240,120 @@ define i7 @v_fshl_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX8-NEXT: v_lshrrev_b16_e32 v0, 7, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i7:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v2.l, 0x7f, v2.l
+; GFX11-NEXT: v_and_b16 v0.h, 0x7f, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 7, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_u32_u16_e32 v3, v2.l
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_u32_u24_e32 v3, 0x2493, v3
+; GFX11-NEXT: v_mul_lo_u16 v1.l, v3.h, 7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_nc_u16 v0.h, v2.l, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.l, 7, v0.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0x7f, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0x7f, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 7, v0
-; GFX9-NEXT: s_movk_i32 s4, 0x2493
+; GFX9-NEXT: v_lshrrev_b16_e32 v1, 1, v1
+; GFX9-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_readfirstlane_b32 s4, v3
+; GFX9-NEXT: s_mul_i32 s5, s4, -7
+; GFX9-NEXT: s_mul_hi_u32 s5, s4, s5
+; GFX9-NEXT: s_add_i32 s4, s4, s5
+; GFX9-NEXT: v_mul_hi_u32 v3, v2, s4
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, -7, v[2:3]
+; GFX9-NEXT: v_add_u32_e32 v3, -7, v2
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_add_u32_e32 v3, -7, v2
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_sub_u16_e32 v3, 6, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, v2, v0
+; GFX9-NEXT: v_and_b32_e32 v2, 0x7f, v3
+; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_u32_u24_sdwa v1, v2, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v3, 7
-; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_sub_u16_e32 v1, v2, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, v1, v0
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, 7, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
; GFX10-NEXT: v_and_b32_e32 v2, 0x7f, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x2493
; GFX10-NEXT: v_and_b32_e32 v1, 0x7f, v1
-; GFX10-NEXT: v_lshlrev_b16 v0, 7, v0
-; GFX10-NEXT: v_mul_u32_u24_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT: v_lshrrev_b16 v1, 1, v1
+; GFX10-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT: v_readfirstlane_b32 s4, v3
+; GFX10-NEXT: s_mul_i32 s5, s4, -7
+; GFX10-NEXT: s_mul_hi_u32 s5, s4, s5
+; GFX10-NEXT: s_add_i32 s4, s4, s5
+; GFX10-NEXT: v_mul_hi_u32 v3, v2, s4
+; GFX10-NEXT: v_mad_u64_u32 v[2:3], s4, v3, -7, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT: v_sub_nc_u16 v3, 6, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX10-NEXT: v_and_b32_e32 v3, 0x7f, v3
+; GFX10-NEXT: v_lshlrev_b16 v0, v2, v0
+; GFX10-NEXT: v_lshrrev_b16 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX10-NEXT: v_mul_lo_u16 v3, v3, 7
-; GFX10-NEXT: v_sub_nc_u16 v1, v2, v3
-; GFX10-NEXT: v_lshlrev_b16 v0, v1, v0
-; GFX10-NEXT: v_lshrrev_b16 v0, 7, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_i7:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0x7f, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0x7f, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 7, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mul_u32_u24_e32 v3, 0x2493, v3
-; GFX11-TRUE16-NEXT: v_mul_lo_u16 v1.l, v3.h, 7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.h, v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 7, v0.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_i7:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0x7f, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0x7f, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 7, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_mul_u32_u24_e32 v3, 0x2493, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_mul_lo_u16 v3, v3, 7
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v1, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, 1, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 7, v0
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-FAKE16-NEXT: s_mul_i32 s1, s0, -7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mul_hi_u32 v5, v2, s0
+; GFX11-FAKE16-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, -7, v3
+; GFX11-FAKE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX11-FAKE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_sub_nc_u16 v3, 6, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0x7f, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v2, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v3, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
@@ -340,45 +383,41 @@ define amdgpu_ps i8 @s_fshl_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_and_b32 s3, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_xor_b32 s2, s2, -1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_and_b32 s3, s2, 7
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_xor_b32 s2, s2, -1
+; GFX9-NEXT: s_and_b32 s3, s2, 7
; GFX9-NEXT: s_lshr_b32 s1, s1, 1
-; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_andn2_b32 s2, 7, s2
; GFX9-NEXT: s_lshl_b32 s0, s0, s3
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
; GFX10-NEXT: s_and_b32 s3, s2, 7
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX10-NEXT: s_xor_b32 s2, s2, -1
+; GFX10-NEXT: s_andn2_b32 s2, 7, s2
; GFX10-NEXT: s_lshr_b32 s1, s1, 1
-; GFX10-NEXT: s_and_b32 s2, s2, 7
; GFX10-NEXT: s_lshl_b32 s0, s0, s3
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_and_b32 s3, s2, 7
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_xor_b32 s2, s2, -1
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_b32 s2, s2, 7
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
}
@@ -405,50 +444,56 @@ define i8 @v_fshl_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX8-NEXT: v_lshrrev_b16_e32 v0, 8, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: v_and_b16 v0.h, v2.l, 7
+; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, v1, v0
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT: v_and_b32_e32 v3, 7, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, v3, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 1
+; GFX9-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX10-NEXT: v_lshlrev_b16 v0, v1, v0
-; GFX10-NEXT: v_lshrrev_b16 v0, 8, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT: v_lshrrev_b16 v1, 1, v1
+; GFX10-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT: v_lshlrev_b16 v0, v2, v0
+; GFX10-NEXT: v_lshrrev_b16 v1, v3, v1
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v2.l, 7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, 1, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v2, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v3, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
@@ -471,6 +516,15 @@ define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i8_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -479,7 +533,6 @@ define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -488,16 +541,6 @@ define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 4
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i8_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 4
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 4)
ret i8 %result
}
@@ -520,6 +563,15 @@ define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i8_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -528,7 +580,6 @@ define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -537,17 +588,6 @@ define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 4, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_i8_4:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_i8_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -578,6 +618,15 @@ define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i8_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 5
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 3
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -586,7 +635,6 @@ define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 3
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -595,16 +643,6 @@ define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i8_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 3
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 5)
ret i8 %result
}
@@ -627,6 +665,15 @@ define i8 @v_fshl_i8_5(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i8_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 3, v0.h
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -635,7 +682,6 @@ define i8 @v_fshl_i8_5(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -644,17 +690,6 @@ define i8 @v_fshl_i8_5(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_i8_5:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 3, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_i8_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -721,27 +756,54 @@ define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshl_v2i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s4, s1, 8
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_lshr_b32 s5, s2, 8
-; GFX9-NEXT: s_and_b32 s6, s2, 7
-; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_xor_b32 s2, s2, -1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 1
-; GFX9-NEXT: s_and_b32 s2, s2, 7
-; GFX9-NEXT: s_lshr_b32 s3, s0, 8
-; GFX9-NEXT: s_lshl_b32 s0, s0, s6
-; GFX9-NEXT: s_lshr_b32 s1, s1, s2
-; GFX9-NEXT: s_or_b32 s0, s0, s1
-; GFX9-NEXT: s_and_b32 s1, s5, 7
+; GFX11-LABEL: s_fshl_v2i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-NEXT: s_and_b32 s6, s2, 7
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_and_b32 s6, s5, 7
+; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-NEXT: s_xor_b32 s5, s5, -1
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_xor_b32 s2, s2, -1
+; GFX11-NEXT: s_lshr_b32 s4, s4, 1
+; GFX11-NEXT: s_and_b32 s5, s5, 7
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_lshl_b32 s3, s3, s6
+; GFX11-NEXT: s_lshr_b32 s4, s4, s5
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_or_b32 s2, s3, s4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_and_b32 s1, s2, 0xff
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshl_v2i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s4, s1, 8
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshr_b32 s5, s2, 8
+; GFX9-NEXT: s_and_b32 s6, s2, 7
+; GFX9-NEXT: s_lshr_b32 s1, s1, 1
+; GFX9-NEXT: s_andn2_b32 s2, 7, s2
+; GFX9-NEXT: s_lshr_b32 s3, s0, 8
+; GFX9-NEXT: s_lshl_b32 s0, s0, s6
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_and_b32 s2, s4, 0xff
-; GFX9-NEXT: s_lshl_b32 s1, s3, s1
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: s_and_b32 s1, s5, 7
; GFX9-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX9-NEXT: s_xor_b32 s3, s5, -1
+; GFX9-NEXT: s_lshl_b32 s1, s3, s1
; GFX9-NEXT: s_lshr_b32 s2, s2, 1
-; GFX9-NEXT: s_and_b32 s3, s3, 7
+; GFX9-NEXT: s_andn2_b32 s3, 7, s5
; GFX9-NEXT: s_lshr_b32 s2, s2, s3
; GFX9-NEXT: s_or_b32 s1, s1, s2
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -749,25 +811,22 @@ define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX9-NEXT: s_lshl_b32 s1, s1, 8
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s4, s1, 8
; GFX10-NEXT: s_lshr_b32 s5, s2, 8
-; GFX10-NEXT: s_and_b32 s6, s2, 7
; GFX10-NEXT: s_and_b32 s4, s4, 0xff
-; GFX10-NEXT: s_lshr_b32 s3, s0, 8
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10-NEXT: s_lshl_b32 s0, s0, s6
-; GFX10-NEXT: s_and_b32 s6, s5, 7
+; GFX10-NEXT: s_and_b32 s6, s2, 7
; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
-; GFX10-NEXT: s_xor_b32 s5, s5, -1
+; GFX10-NEXT: s_lshr_b32 s3, s0, 8
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX10-NEXT: s_xor_b32 s2, s2, -1
+; GFX10-NEXT: s_lshl_b32 s0, s0, s6
+; GFX10-NEXT: s_and_b32 s6, s5, 7
; GFX10-NEXT: s_lshr_b32 s4, s4, 1
-; GFX10-NEXT: s_and_b32 s5, s5, 7
+; GFX10-NEXT: s_andn2_b32 s5, 7, s5
; GFX10-NEXT: s_lshr_b32 s1, s1, 1
-; GFX10-NEXT: s_and_b32 s2, s2, 7
+; GFX10-NEXT: s_andn2_b32 s2, 7, s2
; GFX10-NEXT: s_lshl_b32 s3, s3, s6
; GFX10-NEXT: s_lshr_b32 s4, s4, s5
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
@@ -778,36 +837,6 @@ define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s4, s1, 8
-; GFX11-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-NEXT: s_and_b32 s6, s2, 7
-; GFX11-NEXT: s_and_b32 s4, s4, 0xff
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, s6
-; GFX11-NEXT: s_and_b32 s6, s5, 7
-; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
-; GFX11-NEXT: s_xor_b32 s5, s5, -1
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_xor_b32 s2, s2, -1
-; GFX11-NEXT: s_lshr_b32 s4, s4, 1
-; GFX11-NEXT: s_and_b32 s5, s5, 7
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_b32 s2, s2, 7
-; GFX11-NEXT: s_lshl_b32 s3, s3, s6
-; GFX11-NEXT: s_lshr_b32 s4, s4, s5
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_or_b32 s2, s3, s4
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_and_b32 s1, s2, 0xff
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%amt = bitcast i16 %amt.arg to <2 x i8>
@@ -856,76 +885,107 @@ define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v2i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
+; GFX11-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
+; GFX11-NEXT: v_and_b16 v2.h, 0xff00, v0.l
+; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-NEXT: v_and_b16 v0.h, v0.h, 7
+; GFX11-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.h, v0.h, v1.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b16 v0.l, v2.l, v0.l
+; GFX11-NEXT: v_and_b16 v0.h, 0xff00, v0.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 8, 3
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_lshlrev_b16_e32 v3, v4, v3
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v3
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, v1, v0
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_e32 v6, 7, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, v6, v0
+; GFX9-NEXT: v_mov_b32_e32 v6, 1
+; GFX9-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_and_b32_e32 v1, 7, v5
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, v1, v3
+; GFX9-NEXT: v_xor_b32_e32 v3, -1, v5
+; GFX9-NEXT: v_lshrrev_b16_sdwa v2, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX9-NEXT: v_lshrrev_b16_e32 v2, v3, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX10-NEXT: v_bfe_u32 v4, v2, 8, 3
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_xor_b32_e32 v7, -1, v2
+; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 7, v4
; GFX10-NEXT: v_and_b32_e32 v2, 7, v2
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_lshlrev_b16 v3, v4, v3
+; GFX10-NEXT: v_lshrrev_b16 v1, 1, v1
+; GFX10-NEXT: v_lshrrev_b16 v3, 1, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 7, v6
+; GFX10-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX10-NEXT: v_lshlrev_b16 v4, v4, v5
; GFX10-NEXT: v_lshlrev_b16 v0, v2, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffffff00, v3
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b16 v3, v6, v3
+; GFX10-NEXT: v_lshrrev_b16 v1, v7, v1
+; GFX10-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_and_b32_sdwa v1, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_v2i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff00, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff00, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffffff00, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 8, 3
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v7, -1, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 7, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 7, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, 1, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 1, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 7, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, v4, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v2, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, v6, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v7, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, v5, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v2, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffffff00, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i16 %lhs.arg to <2 x i8>
@@ -997,58 +1057,6 @@ define amdgpu_ps i32 @s_fshl_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshl_v4i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX9-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX9-NEXT: s_and_b32 s0, s2, 7
-; GFX9-NEXT: s_mov_b32 s3, 0xc0c0105
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, s0, v2
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0307
-; GFX9-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30018
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0206
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, s0, v2
-; GFX9-NEXT: v_perm_b32 v0, s1, v0, v3
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30010
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v2, s3
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshl_v4i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xc0c0307
-; GFX10-NEXT: v_mov_b32_e32 v2, 0xc0c0206
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xc0c0004
-; GFX10-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX10-NEXT: v_perm_b32 v2, s1, s0, v2
-; GFX10-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX10-NEXT: v_perm_b32 v3, s1, s0, v3
-; GFX10-NEXT: s_bfe_u32 s0, s2, 0x30018
-; GFX10-NEXT: s_bfe_u32 s1, s2, 0x30010
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, s1, v2
-; GFX10-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX10-NEXT: s_and_b32 s1, s2, 7
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, s1, v3
-; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX10-NEXT: v_perm_b32 v0, v3, v0, 0xc0c0105
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: s_fshl_v4i8:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -1077,6 +1085,108 @@ define amdgpu_ps i32 @s_fshl_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshl_v4i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s11, s2, 7
+; GFX9-NEXT: s_lshr_b32 s3, s0, 8
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshr_b32 s5, s0, 24
+; GFX9-NEXT: s_lshl_b32 s0, s0, s11
+; GFX9-NEXT: s_and_b32 s11, s1, 0xff
+; GFX9-NEXT: s_and_b32 s11, 0xffff, s11
+; GFX9-NEXT: s_lshr_b32 s8, s2, 8
+; GFX9-NEXT: s_lshr_b32 s9, s2, 16
+; GFX9-NEXT: s_lshr_b32 s10, s2, 24
+; GFX9-NEXT: s_lshr_b32 s11, s11, 1
+; GFX9-NEXT: s_andn2_b32 s2, 7, s2
+; GFX9-NEXT: s_lshr_b32 s2, s11, s2
+; GFX9-NEXT: s_lshr_b32 s6, s1, 8
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s8, 7
+; GFX9-NEXT: s_lshl_b32 s2, s3, s2
+; GFX9-NEXT: s_and_b32 s3, s6, 0xff
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX9-NEXT: s_lshr_b32 s3, s3, 1
+; GFX9-NEXT: s_andn2_b32 s6, 7, s8
+; GFX9-NEXT: s_lshr_b32 s3, s3, s6
+; GFX9-NEXT: s_lshr_b32 s7, s1, 16
+; GFX9-NEXT: s_or_b32 s2, s2, s3
+; GFX9-NEXT: s_and_b32 s3, s9, 7
+; GFX9-NEXT: s_lshl_b32 s3, s4, s3
+; GFX9-NEXT: s_and_b32 s4, s7, 0xff
+; GFX9-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX9-NEXT: s_lshr_b32 s4, s4, 1
+; GFX9-NEXT: s_andn2_b32 s6, 7, s9
+; GFX9-NEXT: s_lshr_b32 s4, s4, s6
+; GFX9-NEXT: s_or_b32 s3, s3, s4
+; GFX9-NEXT: s_and_b32 s4, s10, 7
+; GFX9-NEXT: s_lshl_b32 s4, s5, s4
+; GFX9-NEXT: s_lshr_b32 s1, s1, 25
+; GFX9-NEXT: s_andn2_b32 s5, 7, s10
+; GFX9-NEXT: s_and_b32 s2, s2, 0xff
+; GFX9-NEXT: s_lshr_b32 s1, s1, s5
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_or_b32 s1, s4, s1
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s3, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s1, s1, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshl_v4i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s10, s1, 0xff
+; GFX10-NEXT: s_lshr_b32 s6, s1, 8
+; GFX10-NEXT: s_and_b32 s10, 0xffff, s10
+; GFX10-NEXT: s_lshr_b32 s8, s2, 8
+; GFX10-NEXT: s_lshr_b32 s9, s2, 16
+; GFX10-NEXT: s_lshr_b32 s11, s2, 24
+; GFX10-NEXT: s_and_b32 s12, s2, 7
+; GFX10-NEXT: s_lshr_b32 s10, s10, 1
+; GFX10-NEXT: s_andn2_b32 s2, 7, s2
+; GFX10-NEXT: s_lshr_b32 s3, s0, 8
+; GFX10-NEXT: s_lshr_b32 s4, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s0, 24
+; GFX10-NEXT: s_lshl_b32 s0, s0, s12
+; GFX10-NEXT: s_lshr_b32 s2, s10, s2
+; GFX10-NEXT: s_and_b32 s6, s6, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_and_b32 s2, 0xffff, s6
+; GFX10-NEXT: s_lshr_b32 s7, s1, 16
+; GFX10-NEXT: s_and_b32 s6, s8, 7
+; GFX10-NEXT: s_lshr_b32 s2, s2, 1
+; GFX10-NEXT: s_andn2_b32 s8, 7, s8
+; GFX10-NEXT: s_lshl_b32 s3, s3, s6
+; GFX10-NEXT: s_lshr_b32 s2, s2, s8
+; GFX10-NEXT: s_and_b32 s6, s7, 0xff
+; GFX10-NEXT: s_or_b32 s2, s3, s2
+; GFX10-NEXT: s_and_b32 s3, 0xffff, s6
+; GFX10-NEXT: s_and_b32 s6, s9, 7
+; GFX10-NEXT: s_lshr_b32 s3, s3, 1
+; GFX10-NEXT: s_andn2_b32 s7, 7, s9
+; GFX10-NEXT: s_lshl_b32 s4, s4, s6
+; GFX10-NEXT: s_lshr_b32 s3, s3, s7
+; GFX10-NEXT: s_and_b32 s6, s11, 7
+; GFX10-NEXT: s_lshr_b32 s1, s1, 25
+; GFX10-NEXT: s_andn2_b32 s7, 7, s11
+; GFX10-NEXT: s_lshl_b32 s5, s5, s6
+; GFX10-NEXT: s_lshr_b32 s1, s1, s7
+; GFX10-NEXT: s_or_b32 s3, s4, s3
+; GFX10-NEXT: s_and_b32 s2, s2, 0xff
+; GFX10-NEXT: s_or_b32 s1, s5, s1
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, 16
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s1, s1, 24
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
%amt = bitcast i32 %amt.arg to <4 x i8>
@@ -1151,128 +1261,190 @@ define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX8-NEXT: v_or_b32_e32 v0, v3, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v4i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b16 v3.l, 8, v1.l
+; GFX11-NEXT: v_and_b16 v3.h, 0xff00, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 24, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: v_or_b16 v3.l, v3.h, v3.l
+; GFX11-NEXT: v_and_b16 v3.h, v4.l, 7
+; GFX11-NEXT: v_and_b16 v4.l, 0xff00, v0.h
+; GFX11-NEXT: v_and_b16 v1.h, 0xff, v1.h
+; GFX11-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-NEXT: v_and_b16 v1.l, v2.l, 7
+; GFX11-NEXT: v_or_b16 v2.l, v4.l, v6.l
+; GFX11-NEXT: v_and_b16 v4.l, v5.l, 7
+; GFX11-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-NEXT: v_and_b16 v1.h, v2.h, 7
+; GFX11-NEXT: v_lshlrev_b16 v3.l, v3.h, v3.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, v1.l, v0.l
+; GFX11-NEXT: v_lshlrev_b16 v1.l, v4.l, v2.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b16 v2.l, v1.h, v0.h
+; GFX11-NEXT: v_perm_b32 v0, v0, v3, 0xc0c0105
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 8, 3
-; GFX9-NEXT: v_lshlrev_b16_e32 v3, v4, v3
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v0
-; GFX9-NEXT: v_or_b32_sdwa v4, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v5, 7, v2
-; GFX9-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, v5, v4
-; GFX9-NEXT: s_mov_b32 s6, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v3, v4, v3, s6
-; GFX9-NEXT: v_and_b32_sdwa v4, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v5, 7
-; GFX9-NEXT: v_or_b32_sdwa v4, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: v_and_b32_sdwa v6, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_movk_i32 s5, 0xff
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, v6, v4
-; GFX9-NEXT: v_mov_b32_e32 v6, 8
-; GFX9-NEXT: v_and_b32_sdwa v1, v1, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mov_b32_e32 v8, 1
+; GFX9-NEXT: v_xor_b32_e32 v10, -1, v2
+; GFX9-NEXT: v_and_b32_e32 v6, 7, v2
+; GFX9-NEXT: v_lshrrev_b16_sdwa v9, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_and_b32_e32 v10, 7, v10
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v6, v6, v0
+; GFX9-NEXT: v_lshrrev_b16_e32 v9, v10, v9
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_or_b32_e32 v6, v6, v9
+; GFX9-NEXT: v_and_b32_e32 v9, 7, v5
+; GFX9-NEXT: v_xor_b32_e32 v5, -1, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT: v_lshrrev_b16_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_and_b32_e32 v5, 7, v5
+; GFX9-NEXT: v_lshlrev_b16_e32 v3, v9, v3
+; GFX9-NEXT: v_lshrrev_b16_e32 v4, v5, v4
+; GFX9-NEXT: v_mov_b32_e32 v7, 0xff
+; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX9-NEXT: v_mov_b32_e32 v4, 7
+; GFX9-NEXT: v_mov_b32_e32 v9, -1
+; GFX9-NEXT: v_and_b32_sdwa v5, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v8, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_xor_b32_sdwa v10, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT: v_xor_b32_sdwa v2, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b16_e32 v8, 1, v8
+; GFX9-NEXT: v_and_b32_e32 v10, 7, v10
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 25, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT: v_lshlrev_b16_sdwa v5, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshrrev_b16_e32 v8, v10, v8
+; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v8
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_sdwa v1, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, v1, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s6
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v1, 8
+; GFX9-NEXT: v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-NEXT: v_and_or_b32 v1, v6, v7, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xff
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, 7
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffffff00, v0
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_and_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffffff00, v5
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: v_bfe_u32 v7, v2, 8, 3
-; GFX10-NEXT: v_and_b32_e32 v8, 7, v2
-; GFX10-NEXT: v_or_b32_sdwa v6, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-NEXT: v_and_b32_sdwa v9, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX10-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v3, v7, v6
-; GFX10-NEXT: v_lshlrev_b16 v0, v8, v0
-; GFX10-NEXT: v_lshlrev_b16 v1, v9, v1
-; GFX10-NEXT: v_lshlrev_b16 v2, v2, v4
-; GFX10-NEXT: v_perm_b32 v0, v0, v3, 0xc0c0105
-; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX10-NEXT: v_and_b32_e32 v7, 7, v2
+; GFX10-NEXT: v_and_b32_e32 v8, 0xff, v1
+; GFX10-NEXT: v_xor_b32_e32 v9, -1, v2
+; GFX10-NEXT: v_and_b32_e32 v10, 7, v5
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 8, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX10-NEXT: v_lshlrev_b16 v0, v7, v0
+; GFX10-NEXT: v_lshrrev_b16 v7, 1, v8
+; GFX10-NEXT: v_and_b32_e32 v8, 7, v9
+; GFX10-NEXT: v_lshlrev_b16 v3, v10, v3
+; GFX10-NEXT: v_and_b32_e32 v9, 0xff, v11
+; GFX10-NEXT: v_mov_b32_e32 v10, 0xff
+; GFX10-NEXT: v_mov_b32_e32 v11, -1
+; GFX10-NEXT: v_xor_b32_e32 v5, -1, v5
+; GFX10-NEXT: v_mov_b32_e32 v12, 7
+; GFX10-NEXT: v_lshrrev_b16 v9, 1, v9
+; GFX10-NEXT: v_and_b32_sdwa v10, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_xor_b32_sdwa v13, v2, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_xor_b32_sdwa v11, v2, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v5, 7, v5
+; GFX10-NEXT: v_and_b32_sdwa v14, v2, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b16 v10, 1, v10
+; GFX10-NEXT: v_and_b32_e32 v13, 7, v13
+; GFX10-NEXT: v_and_b32_sdwa v2, v2, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 25, v1
+; GFX10-NEXT: v_and_b32_e32 v11, 7, v11
+; GFX10-NEXT: v_lshrrev_b16 v5, v5, v9
+; GFX10-NEXT: v_lshlrev_b16 v4, v14, v4
+; GFX10-NEXT: v_lshrrev_b16 v9, v13, v10
+; GFX10-NEXT: v_lshlrev_b16 v2, v2, v6
+; GFX10-NEXT: v_lshrrev_b16 v1, v11, v1
+; GFX10-NEXT: v_lshrrev_b16 v6, v8, v7
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX10-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v9
+; GFX10-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v4
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX10-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_v4i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff00, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 7
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff00, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v2.l, 7
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v5.l, 7
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.h, 7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v3, 0xc0c0105
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v4, 8, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffffff00, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 24, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffffff00, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v9, 8, v9
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 8, 3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v10, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 7, v7
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v9, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 7, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, v5, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, v7, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, v3, v6
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v4, 0xc0c0105
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v13, -1, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 24, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 7, v8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v6, 1, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 7, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, v8, v3
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v8, -1, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v6, v13, v6
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v13, -1, v10
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 7, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xff, v1
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, -1, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 7, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v7, 1, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 7, v8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 7, v10
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 25, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 7, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v12, 1, v12
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, v9, v4
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v6, v8, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, v10, v5
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v13, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v11, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, v2, v12
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX11-FAKE16-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -1283,74 +1455,77 @@ define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
}
define amdgpu_ps i24 @s_fshl_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt) {
-; GFX6-LABEL: s_fshl_i24:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX6-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX6-NEXT: v_mul_hi_u32 v0, v0, s4
-; GFX6-NEXT: v_readfirstlane_b32 s4, v0
-; GFX6-NEXT: s_add_i32 s3, s3, s4
-; GFX6-NEXT: v_mov_b32_e32 v0, s3
-; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX6-NEXT: s_add_i32 s2, s2, s3
-; GFX6-NEXT: s_cmp_ge_u32 s2, 24
-; GFX6-NEXT: s_cselect_b32 s3, 1, 0
-; GFX6-NEXT: s_sub_i32 s4, s2, 24
-; GFX6-NEXT: s_cmp_lg_u32 s3, 0
-; GFX6-NEXT: s_cselect_b32 s2, s4, s2
-; GFX6-NEXT: s_cmp_ge_u32 s2, 24
-; GFX6-NEXT: s_cselect_b32 s3, 1, 0
-; GFX6-NEXT: s_sub_i32 s4, s2, 24
-; GFX6-NEXT: s_cmp_lg_u32 s3, 0
-; GFX6-NEXT: s_cselect_b32 s2, s4, s2
-; GFX6-NEXT: s_sub_i32 s3, 23, s2
-; GFX6-NEXT: s_bfe_u32 s1, s1, 0x170001
-; GFX6-NEXT: s_lshl_b32 s0, s0, s2
-; GFX6-NEXT: s_lshr_b32 s1, s1, s3
-; GFX6-NEXT: s_or_b32 s0, s0, s1
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshl_i24:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s3, v0
-; GFX8-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
-; GFX8-NEXT: v_readfirstlane_b32 s4, v0
-; GFX8-NEXT: s_add_i32 s3, s3, s4
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT: v_readfirstlane_b32 s3, v0
-; GFX8-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX8-NEXT: s_add_i32 s2, s2, s3
-; GFX8-NEXT: s_cmp_ge_u32 s2, 24
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-NEXT: s_sub_i32 s4, s2, 24
-; GFX8-NEXT: s_cmp_lg_u32 s3, 0
-; GFX8-NEXT: s_cselect_b32 s2, s4, s2
-; GFX8-NEXT: s_cmp_ge_u32 s2, 24
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-NEXT: s_sub_i32 s4, s2, 24
-; GFX8-NEXT: s_cmp_lg_u32 s3, 0
-; GFX8-NEXT: s_cselect_b32 s2, s4, s2
-; GFX8-NEXT: s_sub_i32 s3, 23, s2
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x170001
-; GFX8-NEXT: s_lshl_b32 s0, s0, s2
-; GFX8-NEXT: s_lshr_b32 s1, s1, s3
-; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshl_i24:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT: s_and_b32 s2, s2, 0xffffff
+; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT: v_readfirstlane_b32 s3, v0
+; GCN-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GCN-NEXT: v_mul_hi_u32 v0, v0, s4
+; GCN-NEXT: v_readfirstlane_b32 s4, v0
+; GCN-NEXT: s_add_i32 s3, s3, s4
+; GCN-NEXT: v_mov_b32_e32 v0, s3
+; GCN-NEXT: v_mul_hi_u32 v0, s2, v0
+; GCN-NEXT: v_readfirstlane_b32 s3, v0
+; GCN-NEXT: s_mulk_i32 s3, 0xffe8
+; GCN-NEXT: s_add_i32 s2, s2, s3
+; GCN-NEXT: s_cmp_ge_u32 s2, 24
+; GCN-NEXT: s_cselect_b32 s3, 1, 0
+; GCN-NEXT: s_sub_i32 s4, s2, 24
+; GCN-NEXT: s_cmp_lg_u32 s3, 0
+; GCN-NEXT: s_cselect_b32 s2, s4, s2
+; GCN-NEXT: s_cmp_ge_u32 s2, 24
+; GCN-NEXT: s_cselect_b32 s3, 1, 0
+; GCN-NEXT: s_sub_i32 s4, s2, 24
+; GCN-NEXT: s_cmp_lg_u32 s3, 0
+; GCN-NEXT: s_cselect_b32 s2, s4, s2
+; GCN-NEXT: s_sub_i32 s3, 23, s2
+; GCN-NEXT: s_bfe_u32 s1, s1, 0x170001
+; GCN-NEXT: s_lshl_b32 s0, s0, s2
+; GCN-NEXT: s_lshr_b32 s1, s1, s3
+; GCN-NEXT: s_or_b32 s0, s0, s1
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_bfe_u32 s1, s1, 0x170001
+; GFX11-NEXT: s_sub_i32 s3, 23, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1381,7 +1556,6 @@ define amdgpu_ps i24 @s_fshl_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s3
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1412,44 +1586,6 @@ define amdgpu_ps i24 @s_fshl_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x170001
-; GFX11-NEXT: s_sub_i32 s3, 23, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -1519,6 +1655,42 @@ define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: v_bfe_u32 v1, v1, 1, 23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v2, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1547,7 +1719,6 @@ define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, v3, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, v2, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1575,43 +1746,6 @@ define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX10-NEXT: v_lshrrev_b32_e32 v1, v3, v1
; GFX10-NEXT: v_lshl_or_b32 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: v_bfe_u32 v1, v1, 1, 23
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, v3, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v2, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -1822,339 +1956,462 @@ define amdgpu_ps i48 @s_fshl_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX8-NEXT: v_readfirstlane_b32 s1, v1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_v2i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT: v_mov_b32_e32 v2, s1
+; GFX11-NEXT: v_mov_b32_e32 v4, s3
+; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX11-NEXT: scratch_store_b32 off, v0, off offset:16
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_clause 0x4
+; GFX11-NEXT: scratch_store_b32 off, v0, off
+; GFX11-NEXT: scratch_store_b16 off, v1, off offset:20
+; GFX11-NEXT: scratch_store_b16 off, v2, off offset:4
+; GFX11-NEXT: scratch_store_b32 off, v3, off offset:8
+; GFX11-NEXT: scratch_store_b16 off, v4, off offset:12
+; GFX11-NEXT: s_clause 0xb
+; GFX11-NEXT: scratch_load_u8 v0, off, off offset:21
+; GFX11-NEXT: scratch_load_u8 v1, off, off offset:20
+; GFX11-NEXT: scratch_load_u8 v2, off, off offset:19
+; GFX11-NEXT: scratch_load_u8 v3, off, off offset:11
+; GFX11-NEXT: scratch_load_u8 v4, off, off offset:12
+; GFX11-NEXT: scratch_load_u8 v5, off, off offset:18
+; GFX11-NEXT: scratch_load_u8 v6, off, off offset:5
+; GFX11-NEXT: scratch_load_u8 v7, off, off offset:4
+; GFX11-NEXT: scratch_load_u8 v8, off, off offset:3
+; GFX11-NEXT: scratch_load_u8 v9, off, off offset:13
+; GFX11-NEXT: scratch_load_u8 v10, off, off offset:2
+; GFX11-NEXT: scratch_load_u8 v11, off, off offset:10
+; GFX11-NEXT: s_waitcnt vmcnt(11)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_waitcnt vmcnt(10)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX11-NEXT: s_waitcnt vmcnt(9)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or3_b32 v0, v1, v2, v0
+; GFX11-NEXT: s_waitcnt vmcnt(7)
+; GFX11-NEXT: v_lshl_or_b32 v1, v4, 8, v3
+; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX11-NEXT: s_waitcnt vmcnt(5)
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 8, v7
+; GFX11-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX11-NEXT: v_and_or_b32 v3, 0xffff, s4, v3
+; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: v_or3_b32 v4, v5, v8, v4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshl_or_b32 v5, v11, 24, s1
+; GFX11-NEXT: v_lshl_or_b32 v1, v9, 24, v1
+; GFX11-NEXT: v_mul_hi_u32 v3, 0xaaaaaab, v3
+; GFX11-NEXT: v_lshl_add_u32 v0, v0, 3, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 1, v4
+; GFX11-NEXT: v_alignbit_b32 v1, v8, v1, 1
+; GFX11-NEXT: v_alignbit_b32 v5, s0, v5, 1
+; GFX11-NEXT: v_not_b32_e32 v0, v0
+; GFX11-NEXT: v_and_or_b32 v2, 0xffff, s0, v2
+; GFX11-NEXT: v_lshl_add_u32 v3, v3, 3, s4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_alignbit_b32 v0, v4, v1, v0.l
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_not_b32_e32 v2, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_alignbit_b32 v1, v1, v5, v2.l
+; GFX11-NEXT: s_clause 0x4
+; GFX11-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
+; GFX11-NEXT: scratch_store_b8 off, v3, off offset:28
+; GFX11-NEXT: scratch_store_b8 off, v0, off offset:27
+; GFX11-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
+; GFX11-NEXT: scratch_store_b16 off, v1, off offset:24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: scratch_load_b32 v0, off, off offset:24
+; GFX11-NEXT: scratch_load_u16 v1, off, off offset:28
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i24:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_getpc_b64 s[8:9]
-; GFX9-NEXT: s_mov_b32 s8, s0
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_u32 s8, s8, s6
-; GFX9-NEXT: s_addc_u32 s9, s9, 0
-; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:20
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:16
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:4
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:8
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:12
-; GFX9-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:21
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:5
-; GFX9-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:13
-; GFX9-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:12
-; GFX9-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:20
-; GFX9-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:18
-; GFX9-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:4
-; GFX9-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:11
-; GFX9-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:2
-; GFX9-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:3
-; GFX9-NEXT: buffer_load_ubyte v10, off, s[8:11], 0 offset:10
-; GFX9-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:19
-; GFX9-NEXT: s_and_b32 s5, s4, 0xffff
-; GFX9-NEXT: s_mov_b32 s1, 0xaaaaaab
-; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX9-NEXT: s_and_b32 s3, s0, 0xffff
-; GFX9-NEXT: s_lshl_b32 s2, s2, 8
-; GFX9-NEXT: s_waitcnt vmcnt(11)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_waitcnt vmcnt(10)
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_lshl_or_b32 v5, v5, 16, s5
-; GFX9-NEXT: v_mul_hi_u32 v5, v5, s1
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 8, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v7, v8, 16, s3
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or3_b32 v0, v4, v11, v0
-; GFX9-NEXT: v_mul_hi_u32 v0, v0, s1
-; GFX9-NEXT: v_lshl_or_b32 v8, v10, 24, s2
-; GFX9-NEXT: v_or3_b32 v1, v6, v9, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 24, v3
-; GFX9-NEXT: v_lshl_add_u32 v3, v5, 3, s4
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 3, v11
-; GFX9-NEXT: v_alignbit_b32 v4, s0, v8, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v6, 1, v7
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX9-NEXT: v_alignbit_b32 v2, v9, v2, 1
-; GFX9-NEXT: v_not_b32_e32 v3, v3
-; GFX9-NEXT: v_not_b32_e32 v0, v0
-; GFX9-NEXT: v_alignbit_b32 v3, v6, v4, v3
-; GFX9-NEXT: v_alignbit_b32 v0, v1, v2, v0
-; GFX9-NEXT: buffer_store_byte_d16_hi v3, off, s[8:11], 0 offset:26
-; GFX9-NEXT: buffer_store_short v3, off, s[8:11], 0 offset:24
-; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[8:11], 0 offset:29
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX9-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:27
-; GFX9-NEXT: buffer_store_byte v1, off, s[8:11], 0 offset:28
-; GFX9-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-NEXT: s_lshr_b32 s6, s0, 8
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_lshr_b32 s8, s0, 24
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s6, s6, 8
+; GFX9-NEXT: s_or_b32 s0, s0, s6
+; GFX9-NEXT: s_and_b32 s6, s7, 0xff
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX9-NEXT: s_lshr_b32 s9, s1, 8
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_lshl_b32 s6, s6, 16
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_or_b32 s0, s0, s6
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_and_b32 s6, s9, 0xff
+; GFX9-NEXT: s_or_b32 s1, s8, s1
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s6, s6, 16
+; GFX9-NEXT: s_or_b32 s1, s1, s6
+; GFX9-NEXT: s_lshr_b32 s6, s2, 8
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_lshr_b32 s7, s2, 16
+; GFX9-NEXT: s_lshr_b32 s8, s2, 24
+; GFX9-NEXT: s_and_b32 s2, s2, 0xff
+; GFX9-NEXT: s_lshl_b32 s6, s6, 8
+; GFX9-NEXT: s_or_b32 s2, s2, s6
+; GFX9-NEXT: s_and_b32 s6, s7, 0xff
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX9-NEXT: s_lshr_b32 s9, s3, 8
+; GFX9-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX9-NEXT: s_lshl_b32 s6, s6, 16
+; GFX9-NEXT: s_and_b32 s3, s3, 0xff
+; GFX9-NEXT: s_or_b32 s2, s2, s6
+; GFX9-NEXT: s_lshl_b32 s3, s3, 8
+; GFX9-NEXT: s_and_b32 s6, s9, 0xff
+; GFX9-NEXT: s_or_b32 s3, s8, s3
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX9-NEXT: s_lshl_b32 s6, s6, 16
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX9-NEXT: s_or_b32 s3, s3, s6
+; GFX9-NEXT: s_lshr_b32 s6, s4, 8
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_lshr_b32 s7, s4, 16
+; GFX9-NEXT: s_lshr_b32 s8, s4, 24
+; GFX9-NEXT: s_and_b32 s4, s4, 0xff
+; GFX9-NEXT: s_lshl_b32 s6, s6, 8
+; GFX9-NEXT: s_or_b32 s4, s4, s6
+; GFX9-NEXT: s_and_b32 s6, s7, 0xff
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: s_lshr_b32 s9, s5, 8
+; GFX9-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX9-NEXT: s_lshl_b32 s6, s6, 16
+; GFX9-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: s_or_b32 s4, s4, s6
+; GFX9-NEXT: s_lshl_b32 s5, s5, 8
+; GFX9-NEXT: s_and_b32 s6, s9, 0xff
+; GFX9-NEXT: s_or_b32 s5, s8, s5
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX9-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX9-NEXT: s_lshl_b32 s6, s6, 16
+; GFX9-NEXT: s_or_b32 s5, s5, s6
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_mul_i32 s7, s6, 0xffffffe8
+; GFX9-NEXT: s_mul_hi_u32 s7, s6, s7
+; GFX9-NEXT: s_add_i32 s6, s6, s7
+; GFX9-NEXT: s_mul_hi_u32 s7, s4, s6
+; GFX9-NEXT: s_mulk_i32 s7, 0xffe8
+; GFX9-NEXT: s_add_i32 s4, s4, s7
+; GFX9-NEXT: s_cmp_ge_u32 s4, 24
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_sub_i32 s8, s4, 24
+; GFX9-NEXT: s_cmp_lg_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s4, s8, s4
+; GFX9-NEXT: s_cmp_ge_u32 s4, 24
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_sub_i32 s8, s4, 24
+; GFX9-NEXT: s_cmp_lg_u32 s7, 0
+; GFX9-NEXT: s_cselect_b32 s4, s8, s4
+; GFX9-NEXT: s_sub_i32 s7, 23, s4
+; GFX9-NEXT: s_lshr_b32 s2, s2, 1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s2, s2, s7
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_mul_hi_u32 s2, s5, s6
+; GFX9-NEXT: s_mulk_i32 s2, 0xffe8
+; GFX9-NEXT: s_add_i32 s5, s5, s2
+; GFX9-NEXT: s_cmp_ge_u32 s5, 24
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_sub_i32 s4, s5, 24
+; GFX9-NEXT: s_cmp_lg_u32 s2, 0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s5
+; GFX9-NEXT: s_cmp_ge_u32 s2, 24
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_sub_i32 s5, s2, 24
+; GFX9-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-NEXT: s_cselect_b32 s2, s5, s2
+; GFX9-NEXT: s_sub_i32 s4, 23, s2
+; GFX9-NEXT: s_lshl_b32 s1, s1, s2
+; GFX9-NEXT: s_lshr_b32 s2, s3, 1
+; GFX9-NEXT: s_lshr_b32 s2, s2, s4
+; GFX9-NEXT: s_bfe_u32 s3, s0, 0x80008
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s2, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s3, s3, 8
+; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX9-NEXT: s_or_b32 s2, s2, s3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: s_or_b32 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_bfe_u32 s2, s1, 0x80008
+; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_or_b32 s1, s2, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_v2i24:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_getpc_b64 s[8:9]
-; GFX10-NEXT: s_mov_b32 s8, s0
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bitset0_b32 s11, 21
-; GFX10-NEXT: s_add_u32 s8, s8, s6
-; GFX10-NEXT: s_addc_u32 s9, s9, 0
-; GFX10-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX10-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:16
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_lshr_b32 s6, s0, 8
+; GFX10-NEXT: s_lshr_b32 s7, s0, 16
+; GFX10-NEXT: s_and_b32 s6, s6, 0xff
+; GFX10-NEXT: s_lshr_b32 s8, s0, 24
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s6, s6, 8
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX10-NEXT: s_or_b32 s0, s0, s6
+; GFX10-NEXT: s_and_b32 s6, s7, 0xff
+; GFX10-NEXT: s_lshr_b32 s9, s1, 8
+; GFX10-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_lshl_b32 s6, s6, 16
; GFX10-NEXT: s_lshl_b32 s1, s1, 8
-; GFX10-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX10-NEXT: buffer_store_short v1, off, s[8:11], 0 offset:20
-; GFX10-NEXT: buffer_store_short v2, off, s[8:11], 0 offset:4
-; GFX10-NEXT: buffer_store_dword v3, off, s[8:11], 0 offset:8
-; GFX10-NEXT: buffer_store_short v4, off, s[8:11], 0 offset:12
-; GFX10-NEXT: s_clause 0xb
-; GFX10-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:21
-; GFX10-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:20
-; GFX10-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:18
-; GFX10-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:5
-; GFX10-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:13
-; GFX10-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:12
-; GFX10-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:19
-; GFX10-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:4
-; GFX10-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:11
-; GFX10-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:2
-; GFX10-NEXT: buffer_load_ubyte v10, off, s[8:11], 0 offset:3
-; GFX10-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:10
-; GFX10-NEXT: s_waitcnt vmcnt(11)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_waitcnt vmcnt(10)
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: s_waitcnt vmcnt(9)
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT: s_waitcnt vmcnt(5)
-; GFX10-NEXT: v_or3_b32 v0, v1, v6, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX10-NEXT: s_waitcnt vmcnt(3)
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 8, v8
-; GFX10-NEXT: v_and_or_b32 v2, 0xffff, s4, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v7
-; GFX10-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX10-NEXT: s_waitcnt vmcnt(2)
-; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX10-NEXT: v_mul_hi_u32 v2, 0xaaaaaab, v2
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_or3_b32 v1, v5, v10, v1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX10-NEXT: v_lshl_or_b32 v3, v4, 24, v3
-; GFX10-NEXT: v_lshl_add_u32 v0, v0, 3, v6
-; GFX10-NEXT: v_and_or_b32 v4, 0xffff, s0, v7
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX10-NEXT: v_lshl_add_u32 v2, v2, 3, s4
-; GFX10-NEXT: v_alignbit_b32 v3, v10, v3, 1
-; GFX10-NEXT: v_not_b32_e32 v0, v0
-; GFX10-NEXT: v_alignbit_b32 v5, s0, v5, 1
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 1, v4
-; GFX10-NEXT: v_not_b32_e32 v2, v2
-; GFX10-NEXT: v_alignbit_b32 v0, v1, v3, v0
-; GFX10-NEXT: v_alignbit_b32 v1, v4, v5, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: buffer_store_byte_d16_hi v0, off, s[8:11], 0 offset:29
-; GFX10-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:27
-; GFX10-NEXT: buffer_store_byte_d16_hi v1, off, s[8:11], 0 offset:26
-; GFX10-NEXT: buffer_store_short v1, off, s[8:11], 0 offset:24
-; GFX10-NEXT: buffer_store_byte v2, off, s[8:11], 0 offset:28
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX10-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_and_b32 s7, s9, 0xff
+; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT: s_or_b32 s1, s8, s1
+; GFX10-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX10-NEXT: s_or_b32 s0, s0, s6
+; GFX10-NEXT: s_lshr_b32 s6, s2, 8
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshl_b32 s7, s7, 16
+; GFX10-NEXT: s_and_b32 s6, s6, 0xff
+; GFX10-NEXT: s_or_b32 s1, s1, s7
+; GFX10-NEXT: s_lshr_b32 s7, s2, 16
+; GFX10-NEXT: s_lshr_b32 s8, s2, 24
+; GFX10-NEXT: s_and_b32 s2, s2, 0xff
+; GFX10-NEXT: s_lshl_b32 s6, s6, 8
+; GFX10-NEXT: s_lshr_b32 s9, s3, 8
+; GFX10-NEXT: s_or_b32 s2, s2, s6
+; GFX10-NEXT: s_and_b32 s6, s7, 0xff
+; GFX10-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX10-NEXT: s_lshl_b32 s6, s6, 16
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_and_b32 s7, s9, 0xff
+; GFX10-NEXT: s_or_b32 s3, s8, s3
+; GFX10-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX10-NEXT: s_or_b32 s2, s2, s6
+; GFX10-NEXT: s_lshr_b32 s6, s4, 8
+; GFX10-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX10-NEXT: s_lshl_b32 s7, s7, 16
+; GFX10-NEXT: s_and_b32 s6, s6, 0xff
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT: s_or_b32 s3, s3, s7
+; GFX10-NEXT: s_lshr_b32 s7, s4, 16
+; GFX10-NEXT: s_lshr_b32 s8, s4, 24
+; GFX10-NEXT: s_and_b32 s4, s4, 0xff
+; GFX10-NEXT: s_lshl_b32 s6, s6, 8
+; GFX10-NEXT: s_lshr_b32 s9, s5, 8
+; GFX10-NEXT: s_or_b32 s4, s4, s6
+; GFX10-NEXT: s_and_b32 s6, s7, 0xff
+; GFX10-NEXT: v_readfirstlane_b32 s7, v0
+; GFX10-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX10-NEXT: s_lshl_b32 s6, s6, 16
+; GFX10-NEXT: s_and_b32 s5, s5, 0xff
+; GFX10-NEXT: s_or_b32 s4, s4, s6
+; GFX10-NEXT: s_mul_i32 s6, s7, 0xffffffe8
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_mul_hi_u32 s6, s7, s6
+; GFX10-NEXT: s_or_b32 s5, s8, s5
+; GFX10-NEXT: s_add_i32 s7, s7, s6
+; GFX10-NEXT: s_and_b32 s8, s9, 0xff
+; GFX10-NEXT: s_mul_hi_u32 s6, s4, s7
+; GFX10-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX10-NEXT: s_mulk_i32 s6, 0xffe8
+; GFX10-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX10-NEXT: s_lshl_b32 s8, s8, 16
+; GFX10-NEXT: s_add_i32 s4, s4, s6
+; GFX10-NEXT: s_or_b32 s5, s5, s8
+; GFX10-NEXT: s_cmp_ge_u32 s4, 24
+; GFX10-NEXT: s_cselect_b32 s6, 1, 0
+; GFX10-NEXT: s_sub_i32 s8, s4, 24
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
+; GFX10-NEXT: s_cselect_b32 s4, s8, s4
+; GFX10-NEXT: s_cmp_ge_u32 s4, 24
+; GFX10-NEXT: s_cselect_b32 s6, 1, 0
+; GFX10-NEXT: s_sub_i32 s8, s4, 24
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
+; GFX10-NEXT: s_cselect_b32 s4, s8, s4
+; GFX10-NEXT: s_lshr_b32 s2, s2, 1
+; GFX10-NEXT: s_sub_i32 s6, 23, s4
+; GFX10-NEXT: s_lshl_b32 s0, s0, s4
+; GFX10-NEXT: s_mul_hi_u32 s4, s5, s7
+; GFX10-NEXT: s_lshr_b32 s2, s2, s6
+; GFX10-NEXT: s_mulk_i32 s4, 0xffe8
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_add_i32 s5, s5, s4
+; GFX10-NEXT: s_cmp_ge_u32 s5, 24
+; GFX10-NEXT: s_cselect_b32 s2, 1, 0
+; GFX10-NEXT: s_sub_i32 s4, s5, 24
+; GFX10-NEXT: s_cmp_lg_u32 s2, 0
+; GFX10-NEXT: s_cselect_b32 s2, s4, s5
+; GFX10-NEXT: s_cmp_ge_u32 s2, 24
+; GFX10-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-NEXT: s_sub_i32 s5, s2, 24
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: s_cselect_b32 s2, s5, s2
+; GFX10-NEXT: s_lshr_b32 s3, s3, 1
+; GFX10-NEXT: s_sub_i32 s4, 23, s2
+; GFX10-NEXT: s_lshl_b32 s1, s1, s2
+; GFX10-NEXT: s_lshr_b32 s2, s3, s4
+; GFX10-NEXT: s_and_b32 s3, s0, 0xff
+; GFX10-NEXT: s_or_b32 s1, s1, s2
+; GFX10-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX10-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_lshl_b32 s0, s0, 16
+; GFX10-NEXT: s_or_b32 s2, s3, s2
+; GFX10-NEXT: s_and_b32 s3, s1, 0xff
+; GFX10-NEXT: s_or_b32 s0, s2, s0
+; GFX10-NEXT: s_lshl_b32 s2, s3, 24
+; GFX10-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX10-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_fshl_v2i24:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v0, off offset:16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_clause 0x4
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v0, off
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v1, off offset:20
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v2, off offset:4
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v3, off offset:8
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v4, off offset:12
-; GFX11-TRUE16-NEXT: s_clause 0xb
-; GFX11-TRUE16-NEXT: scratch_load_u8 v0, off, off offset:21
-; GFX11-TRUE16-NEXT: scratch_load_u8 v1, off, off offset:20
-; GFX11-TRUE16-NEXT: scratch_load_u8 v2, off, off offset:19
-; GFX11-TRUE16-NEXT: scratch_load_u8 v3, off, off offset:11
-; GFX11-TRUE16-NEXT: scratch_load_u8 v4, off, off offset:12
-; GFX11-TRUE16-NEXT: scratch_load_u8 v5, off, off offset:18
-; GFX11-TRUE16-NEXT: scratch_load_u8 v6, off, off offset:5
-; GFX11-TRUE16-NEXT: scratch_load_u8 v7, off, off offset:4
-; GFX11-TRUE16-NEXT: scratch_load_u8 v8, off, off offset:3
-; GFX11-TRUE16-NEXT: scratch_load_u8 v9, off, off offset:13
-; GFX11-TRUE16-NEXT: scratch_load_u8 v10, off, off offset:2
-; GFX11-TRUE16-NEXT: scratch_load_u8 v11, off, off offset:10
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or3_b32 v0, v1, v2, v0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v4, 8, v3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 8, v7
-; GFX11-TRUE16-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff, s4, v3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_or3_b32 v4, v5, v8, v4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v9, 24, v1
-; GFX11-TRUE16-NEXT: v_mul_hi_u32 v3, 0xaaaaaab, v3
-; GFX11-TRUE16-NEXT: v_lshl_add_u32 v0, v0, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 1, v4
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v8, v1, 1
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v5, s0, v5, 1
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v0, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff, s0, v2
-; GFX11-TRUE16-NEXT: v_lshl_add_u32 v3, v3, 3, s4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v4, v1, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v5, v2.l
-; GFX11-TRUE16-NEXT: s_clause 0x4
-; GFX11-TRUE16-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
-; GFX11-TRUE16-NEXT: scratch_store_b8 off, v3, off offset:28
-; GFX11-TRUE16-NEXT: scratch_store_b8 off, v0, off offset:27
-; GFX11-TRUE16-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v1, off offset:24
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, off offset:24
-; GFX11-TRUE16-NEXT: scratch_load_u16 v1, off, off offset:28
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: s_fshl_v2i24:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v0, off offset:16
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s0, 24
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 16
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-FAKE16-NEXT: s_clause 0x4
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v0, off
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v1, off offset:20
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v2, off offset:4
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v3, off offset:8
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v4, off offset:12
-; GFX11-FAKE16-NEXT: s_clause 0xb
-; GFX11-FAKE16-NEXT: scratch_load_u8 v0, off, off offset:21
-; GFX11-FAKE16-NEXT: scratch_load_u8 v1, off, off offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u8 v2, off, off offset:19
-; GFX11-FAKE16-NEXT: scratch_load_u8 v3, off, off offset:11
-; GFX11-FAKE16-NEXT: scratch_load_u8 v4, off, off offset:12
-; GFX11-FAKE16-NEXT: scratch_load_u8 v5, off, off offset:18
-; GFX11-FAKE16-NEXT: scratch_load_u8 v6, off, off offset:5
-; GFX11-FAKE16-NEXT: scratch_load_u8 v7, off, off offset:4
-; GFX11-FAKE16-NEXT: scratch_load_u8 v8, off, off offset:3
-; GFX11-FAKE16-NEXT: scratch_load_u8 v9, off, off offset:13
-; GFX11-FAKE16-NEXT: scratch_load_u8 v10, off, off offset:2
-; GFX11-FAKE16-NEXT: scratch_load_u8 v11, off, off offset:10
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or3_b32 v0, v1, v2, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 8, v3
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(4)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 8, v7
-; GFX11-FAKE16-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff, s4, v3
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_or3_b32 v4, v5, v8, v4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v9, 24, v1
-; GFX11-FAKE16-NEXT: v_mul_hi_u32 v3, 0xaaaaaab, v3
-; GFX11-FAKE16-NEXT: v_lshl_add_u32 v0, v0, 3, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v10
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 1, v4
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v8, v1, 1
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v5, s0, v5, 1
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v0, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff, s0, v2
-; GFX11-FAKE16-NEXT: v_lshl_add_u32 v3, v3, 3, s4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v4, v1, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 1, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v2, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v1, v5, v2
-; GFX11-FAKE16-NEXT: s_clause 0x4
-; GFX11-FAKE16-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
-; GFX11-FAKE16-NEXT: scratch_store_b8 off, v3, off offset:28
-; GFX11-FAKE16-NEXT: scratch_store_b8 off, v0, off offset:27
-; GFX11-FAKE16-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v1, off offset:24
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, off offset:24
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, off offset:28
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s9, 0xff
+; GFX11-FAKE16-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s8, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s6
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s2, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s7, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s2, 24
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s3, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s9, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s8, s3
+; GFX11-FAKE16-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s6
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s4, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s7, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s3, s7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 24
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s5, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s4, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s5, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s4, s6
+; GFX11-FAKE16-NEXT: s_mul_i32 s6, s7, 0xffffffe8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s6, s7, s6
+; GFX11-FAKE16-NEXT: s_or_b32 s5, s8, s5
+; GFX11-FAKE16-NEXT: s_add_i32 s7, s7, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s9, 0xff
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s6, s4, s7
+; GFX11-FAKE16-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX11-FAKE16-NEXT: s_mulk_i32 s6, 0xffe8
+; GFX11-FAKE16-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s8, 16
+; GFX11-FAKE16-NEXT: s_add_i32 s4, s4, s6
+; GFX11-FAKE16-NEXT: s_or_b32 s5, s5, s8
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s8, s4, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, s8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s8, s4, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, s8, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-FAKE16-NEXT: s_sub_i32 s6, 23, s4
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s4, s5, s7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
+; GFX11-FAKE16-NEXT: s_mulk_i32 s4, 0xffe8
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_add_i32 s5, s5, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s5, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s4, s5, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, s4, s5
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s5, s2, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, s5, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-FAKE16-NEXT: s_sub_i32 s4, 23, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s3, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s0, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s3, s2
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s3, 24
+; GFX11-FAKE16-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s3
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i48 %lhs.arg to <2 x i24>
%rhs = bitcast i48 %rhs.arg to <2 x i24>
@@ -2263,6 +2520,58 @@ define <2 x i24> @v_fshl_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v2i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_bfe_u32 v2, v2, 1, 23
+; GFX11-NEXT: v_bfe_u32 v3, v3, 1, 23
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
+; GFX11-NEXT: v_readfirstlane_b32 s0, v6
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
+; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffffff, v7
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v7, v3
+; GFX11-NEXT: v_lshl_or_b32 v1, v1, v5, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2306,7 +2615,6 @@ define <2 x i24> @v_fshl_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v3, v4, v3
; GFX9-NEXT: v_lshl_or_b32 v1, v1, v2, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2349,86 +2657,33 @@ define <2 x i24> @v_fshl_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v7, v3
; GFX10-NEXT: v_lshl_or_b32 v1, v1, v4, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_v2i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_bfe_u32 v2, v2, 1, 23
-; GFX11-NEXT: v_bfe_u32 v3, v3, 1, 23
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
-; GFX11-NEXT: v_readfirstlane_b32 s0, v6
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
-; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
-; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffffff, v7
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v7, v3
-; GFX11-NEXT: v_lshl_or_b32 v1, v1, v5, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i24> @llvm.fshl.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
ret <2 x i24> %result
}
define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GFX6-LABEL: s_fshl_i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: s_not_b32 s1, s2
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX6-NEXT: s_lshr_b32 s0, s0, 1
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshl_i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: s_not_b32 s1, s2
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX8-NEXT: s_lshr_b32 s0, s0, 1
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshl_i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: s_not_b32 s1, s2
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GCN-NEXT: s_lshr_b32 s0, s0, 1
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_not_b32 s2, s2
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -2439,7 +2694,6 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -2448,19 +2702,6 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_fshl_i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: s_fshl_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -2476,77 +2717,59 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
}
define amdgpu_ps i32 @s_fshl_i32_5(i32 inreg %lhs, i32 inreg %rhs) {
-; GFX6-LABEL: s_fshl_i32_5:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 27
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshl_i32_5:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 27
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshl_i32_5:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 27
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i32_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 27
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i32_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 27
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i32_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 27
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i32_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 5)
ret i32 %result
}
define amdgpu_ps i32 @s_fshl_i32_8(i32 inreg %lhs, i32 inreg %rhs) {
-; GFX6-LABEL: s_fshl_i32_8:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 24
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshl_i32_8:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 24
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshl_i32_8:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 24
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i32_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 24
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i32_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 24
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i32_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 24
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i32_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 24
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 8)
ret i32 %result
}
@@ -2561,16 +2784,15 @@ define i32 @v_fshl_i32(i32 %lhs, i32 %rhs, i32 %amt) {
; GCN-NEXT: v_alignbit_b32 v0, v0, v1, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_fshl_i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
+; GFX11-LABEL: v_fshl_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v1, v0, v1, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: v_not_b32_e32 v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-FAKE16-LABEL: v_fshl_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2617,24 +2839,23 @@ define i32 @v_fshl_i32_8(i32 %lhs, i32 %rhs) {
}
define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt) {
-; GFX6-LABEL: v_fshl_i32_ssv:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v1, s0, v1, 1
-; GFX6-NEXT: s_lshr_b32 s0, s0, 1
-; GFX6-NEXT: v_not_b32_e32 v0, v0
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v1, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: v_fshl_i32_ssv:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_alignbit_b32 v1, s0, v1, 1
-; GFX8-NEXT: s_lshr_b32 s0, s0, 1
-; GFX8-NEXT: v_not_b32_e32 v0, v0
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v1, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: v_fshl_i32_ssv:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_alignbit_b32 v1, s0, v1, 1
+; GCN-NEXT: s_lshr_b32 s0, s0, 1
+; GCN-NEXT: v_not_b32_e32 v0, v0
+; GCN-NEXT: v_alignbit_b32 v0, s0, v1, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i32_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v1, s0, s1, 1
+; GFX11-NEXT: v_not_b32_e32 v0, v0
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v1, v0.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i32_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, s1
@@ -2643,7 +2864,6 @@ define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
; GFX9-NEXT: v_not_b32_e32 v0, v0
; GFX9-NEXT: v_alignbit_b32 v0, s0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i32_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v1, s0, s1, 1
@@ -2651,16 +2871,6 @@ define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
; GFX10-NEXT: s_lshr_b32 s0, s0, 1
; GFX10-NEXT: v_alignbit_b32 v0, s0, v1, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshl_i32_ssv:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s0, s1, 1
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v0, v0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v1, v0.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshl_i32_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s0, s1, 1
@@ -2675,24 +2885,24 @@ define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
}
define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt) {
-; GFX6-LABEL: v_fshl_i32_svs:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_not_b32 s1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX6-NEXT: s_lshr_b32 s0, s0, 1
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: v_fshl_i32_svs:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_not_b32 s1, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX8-NEXT: s_lshr_b32 s0, s0, 1
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: v_fshl_i32_svs:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_not_b32 s1, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GCN-NEXT: s_lshr_b32 s0, s0, 1
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i32_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_not_b32 s1, s1
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i32_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_not_b32 s1, s1
@@ -2701,7 +2911,6 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i32_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, 1
@@ -2709,17 +2918,6 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX10-NEXT: s_not_b32 s1, s1
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshl_i32_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s1, s1
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshl_i32_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
@@ -2734,26 +2932,25 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
}
define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GFX6-LABEL: v_fshl_i32_vss:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: s_not_b32 s1, s2
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX6-NEXT: s_lshr_b32 s0, s0, 1
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: v_fshl_i32_vss:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: s_not_b32 s1, s2
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX8-NEXT: s_lshr_b32 s0, s0, 1
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: v_fshl_i32_vss:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: s_not_b32 s1, s2
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GCN-NEXT: s_lshr_b32 s0, s0, 1
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i32_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_not_b32 s2, s2
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i32_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -2763,7 +2960,6 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i32_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -2771,17 +2967,6 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
; GFX10-NEXT: s_not_b32 s1, s2
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshl_i32_vss:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshl_i32_vss:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -2796,32 +2981,32 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
}
define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
-; GFX6-LABEL: v_fshl_v2i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_alignbit_b32 v2, v0, v2, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX6-NEXT: v_not_b32_e32 v4, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v2, v4
-; GFX6-NEXT: v_alignbit_b32 v2, v1, v3, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX6-NEXT: v_not_b32_e32 v3, v5
-; GFX6-NEXT: v_alignbit_b32 v1, v1, v2, v3
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: v_fshl_v2i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_alignbit_b32 v2, v0, v2, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX8-NEXT: v_not_b32_e32 v4, v4
-; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, v4
-; GFX8-NEXT: v_alignbit_b32 v2, v1, v3, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX8-NEXT: v_not_b32_e32 v3, v5
-; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, v3
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GCN-LABEL: v_fshl_v2i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_alignbit_b32 v2, v0, v2, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GCN-NEXT: v_not_b32_e32 v4, v4
+; GCN-NEXT: v_alignbit_b32 v0, v0, v2, v4
+; GCN-NEXT: v_alignbit_b32 v2, v1, v3, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT: v_not_b32_e32 v3, v5
+; GCN-NEXT: v_alignbit_b32 v1, v1, v2, v3
+; GCN-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v2i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v2, v0, v2, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: v_alignbit_b32 v3, v1, v3, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX11-NEXT: v_not_b32_e32 v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
+; GFX11-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2834,7 +3019,6 @@ define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
; GFX9-NEXT: v_not_b32_e32 v3, v5
; GFX9-NEXT: v_alignbit_b32 v1, v1, v2, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v2i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2847,21 +3031,6 @@ define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
; GFX10-NEXT: v_alignbit_b32 v0, v0, v2, v4
; GFX10-NEXT: v_alignbit_b32 v1, v1, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_v2i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v0, v2, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v1, v3, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_v2i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2872,48 +3041,48 @@ define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v0, v2, v4
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v1, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
- %result = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt)
- ret <2 x i32> %result
-}
-
-define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
-; GFX6-LABEL: v_fshl_v3i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_alignbit_b32 v3, v0, v3, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX6-NEXT: v_not_b32_e32 v6, v6
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v3, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v1, v4, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX6-NEXT: v_not_b32_e32 v4, v7
-; GFX6-NEXT: v_alignbit_b32 v1, v1, v3, v4
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v5, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX6-NEXT: v_not_b32_e32 v4, v8
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v3, v4
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: v_fshl_v3i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_alignbit_b32 v3, v0, v3, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX8-NEXT: v_not_b32_e32 v6, v6
-; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, v6
-; GFX8-NEXT: v_alignbit_b32 v3, v1, v4, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX8-NEXT: v_not_b32_e32 v4, v7
-; GFX8-NEXT: v_alignbit_b32 v1, v1, v3, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v2, v5, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX8-NEXT: v_not_b32_e32 v4, v8
-; GFX8-NEXT: v_alignbit_b32 v2, v2, v3, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v0, v2, v4
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v1, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %result = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt)
+ ret <2 x i32> %result
+}
+
+define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
+; GCN-LABEL: v_fshl_v3i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_alignbit_b32 v3, v0, v3, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GCN-NEXT: v_not_b32_e32 v6, v6
+; GCN-NEXT: v_alignbit_b32 v0, v0, v3, v6
+; GCN-NEXT: v_alignbit_b32 v3, v1, v4, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT: v_not_b32_e32 v4, v7
+; GCN-NEXT: v_alignbit_b32 v1, v1, v3, v4
+; GCN-NEXT: v_alignbit_b32 v3, v2, v5, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GCN-NEXT: v_not_b32_e32 v4, v8
+; GCN-NEXT: v_alignbit_b32 v2, v2, v3, v4
+; GCN-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v3i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v3, v0, v3, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: v_not_b32_e32 v6, v6
+; GFX11-NEXT: v_alignbit_b32 v4, v1, v4, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX11-NEXT: v_not_b32_e32 v7, v7
+; GFX11-NEXT: v_alignbit_b32 v5, v2, v5, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX11-NEXT: v_not_b32_e32 v8, v8
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
+; GFX11-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v3i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2930,7 +3099,6 @@ define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
; GFX9-NEXT: v_not_b32_e32 v4, v8
; GFX9-NEXT: v_alignbit_b32 v2, v2, v3, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v3i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2947,25 +3115,6 @@ define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
; GFX10-NEXT: v_alignbit_b32 v1, v1, v4, v7
; GFX10-NEXT: v_alignbit_b32 v2, v2, v5, v8
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_v3i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v0, v3, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v6
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v4, v1, v4, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v7
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v5, v2, v5, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v8, v8
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_v3i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2988,48 +3137,48 @@ define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
}
define <4 x i32> @v_fshl_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
-; GFX6-LABEL: v_fshl_v4i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_alignbit_b32 v4, v0, v4, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX6-NEXT: v_not_b32_e32 v8, v8
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v4, v8
-; GFX6-NEXT: v_alignbit_b32 v4, v1, v5, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX6-NEXT: v_not_b32_e32 v5, v9
-; GFX6-NEXT: v_alignbit_b32 v1, v1, v4, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v2, v6, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX6-NEXT: v_not_b32_e32 v5, v10
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v7, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 1, v3
-; GFX6-NEXT: v_not_b32_e32 v5, v11
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, v5
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: v_fshl_v4i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_alignbit_b32 v4, v0, v4, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX8-NEXT: v_not_b32_e32 v8, v8
-; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, v8
-; GFX8-NEXT: v_alignbit_b32 v4, v1, v5, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX8-NEXT: v_not_b32_e32 v5, v9
-; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, v5
-; GFX8-NEXT: v_alignbit_b32 v4, v2, v6, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX8-NEXT: v_not_b32_e32 v5, v10
-; GFX8-NEXT: v_alignbit_b32 v2, v2, v4, v5
-; GFX8-NEXT: v_alignbit_b32 v4, v3, v7, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 1, v3
-; GFX8-NEXT: v_not_b32_e32 v5, v11
-; GFX8-NEXT: v_alignbit_b32 v3, v3, v4, v5
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GCN-LABEL: v_fshl_v4i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_alignbit_b32 v4, v0, v4, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GCN-NEXT: v_not_b32_e32 v8, v8
+; GCN-NEXT: v_alignbit_b32 v0, v0, v4, v8
+; GCN-NEXT: v_alignbit_b32 v4, v1, v5, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT: v_not_b32_e32 v5, v9
+; GCN-NEXT: v_alignbit_b32 v1, v1, v4, v5
+; GCN-NEXT: v_alignbit_b32 v4, v2, v6, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GCN-NEXT: v_not_b32_e32 v5, v10
+; GCN-NEXT: v_alignbit_b32 v2, v2, v4, v5
+; GCN-NEXT: v_alignbit_b32 v4, v3, v7, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GCN-NEXT: v_not_b32_e32 v5, v11
+; GCN-NEXT: v_alignbit_b32 v3, v3, v4, v5
+; GCN-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v4i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v4, v0, v4, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: v_not_b32_e32 v8, v8
+; GFX11-NEXT: v_alignbit_b32 v5, v1, v5, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX11-NEXT: v_not_b32_e32 v9, v9
+; GFX11-NEXT: v_alignbit_b32 v6, v2, v6, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX11-NEXT: v_not_b32_e32 v10, v10
+; GFX11-NEXT: v_alignbit_b32 v7, v3, v7, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX11-NEXT: v_not_b32_e32 v11, v11
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
+; GFX11-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
+; GFX11-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v4i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3050,7 +3199,6 @@ define <4 x i32> @v_fshl_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
; GFX9-NEXT: v_not_b32_e32 v5, v11
; GFX9-NEXT: v_alignbit_b32 v3, v3, v4, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v4i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3071,29 +3219,6 @@ define <4 x i32> @v_fshl_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
; GFX10-NEXT: v_alignbit_b32 v2, v2, v6, v10
; GFX10-NEXT: v_alignbit_b32 v3, v3, v7, v11
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_v4i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v4, v0, v4, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v8, v8
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v5, v1, v5, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v9
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v6, v2, v6, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v10, v10
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v7, v3, v7, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 1, v3
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v11
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_v4i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3141,6 +3266,17 @@ define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s3, s2, 15
+; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3151,7 +3287,6 @@ define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3162,18 +3297,6 @@ define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_b32 s3, s2, 15
-; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 %amt)
ret i16 %result
}
@@ -3194,6 +3317,14 @@ define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i16_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_lshr_b32 s1, s1, 12
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3201,7 +3332,6 @@ define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 12
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3209,15 +3339,6 @@ define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 12
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i16_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_lshr_b32 s1, s1, 12
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 4)
ret i16 %result
}
@@ -3238,6 +3359,14 @@ define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i16_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 5
+; GFX11-NEXT: s_lshr_b32 s1, s1, 11
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3245,7 +3374,6 @@ define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 11
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3253,15 +3381,6 @@ define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 11
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i16_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_lshr_b32 s1, s1, 11
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 5)
ret i16 %result
}
@@ -3291,6 +3410,19 @@ define i16 @v_fshl_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b16 v0.h, v2.l, -1
+; GFX11-NEXT: v_and_b16 v1.h, v2.l, 15
+; GFX11-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, v0.h, v1.l
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3302,7 +3434,6 @@ define i16 @v_fshl_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3314,21 +3445,6 @@ define i16 @v_fshl_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX10-NEXT: v_lshrrev_b16 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, -1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 15
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3363,6 +3479,14 @@ define i16 @v_fshl_i16_4(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i16_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 12, v1.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3370,7 +3494,6 @@ define i16 @v_fshl_i16_4(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 12, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3378,16 +3501,6 @@ define i16 @v_fshl_i16_4(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 12, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_i16_4:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 12, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_i16_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3417,6 +3530,14 @@ define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i16_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 11, v1.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3424,7 +3545,6 @@ define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 11, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3432,16 +3552,6 @@ define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 11, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_i16_5:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 11, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshl_i16_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3457,66 +3567,73 @@ define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
define amdgpu_ps half @v_fshl_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt) {
; GFX6-LABEL: v_fshl_i16_ssv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: v_and_b32_e32 v1, 15, v0
+; GFX6-NEXT: v_xor_b32_e32 v0, -1, v0
; GFX6-NEXT: v_and_b32_e32 v0, 15, v0
-; GFX6-NEXT: v_lshl_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshl_b32_e32 v1, s0, v1
+; GFX6-NEXT: s_bfe_u32 s0, s1, 0xf0001
+; GFX6-NEXT: v_lshr_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i16_ssv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX8-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 1
-; GFX8-NEXT: v_lshrrev_b16_e64 v1, v1, s1
-; GFX8-NEXT: v_lshlrev_b16_e64 v0, v0, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v0
+; GFX8-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT: v_lshlrev_b16_e64 v1, v1, s0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s1
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 1
+; GFX8-NEXT: v_lshrrev_b16_e64 v0, v0, s0
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i16_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b16 v0.h, v0.l, -1
+; GFX11-NEXT: v_and_b16 v0.l, v0.l, 15
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.l, s0
+; GFX11-NEXT: v_lshrrev_b16 v0.h, v0.h, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i16_ssv:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX9-NEXT: s_lshr_b32 s1, s1, 1
-; GFX9-NEXT: v_lshrrev_b16_e64 v1, v1, s1
-; GFX9-NEXT: v_lshlrev_b16_e64 v0, v0, s0
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_and_b32_e32 v1, 15, v0
+; GFX9-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT: v_lshlrev_b16_e64 v1, v1, s0
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s1
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 1
+; GFX9-NEXT: v_lshrrev_b16_e64 v0, v0, s0
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT: v_and_b32_e32 v0, 15, v0
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX10-NEXT: v_lshlrev_b16 v0, v0, s0
; GFX10-NEXT: s_lshr_b32 s1, s1, 1
+; GFX10-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX10-NEXT: v_lshlrev_b16 v0, v0, s0
; GFX10-NEXT: v_lshrrev_b16 v1, v1, s1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshl_i16_ssv:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v0.l, -1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
-; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshl_i16_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 15, v0
; GFX11-FAKE16-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v0, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v0, s0
; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v1, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
@@ -3529,34 +3646,44 @@ define amdgpu_ps half @v_fshl_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
define amdgpu_ps half @v_fshl_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt) {
; GFX6-LABEL: v_fshl_i16_svs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s2, s1, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 1, 15
+; GFX6-NEXT: s_lshl_b32 s0, s0, s2
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s1, v0
; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX6-NEXT: s_and_b32 s0, s1, 15
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i16_svs:
; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s2, s1, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
; GFX8-NEXT: v_lshrrev_b16_e32 v0, 1, v0
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
-; GFX8-NEXT: s_and_b32 s1, s1, 15
-; GFX8-NEXT: v_lshrrev_b16_e32 v0, s2, v0
-; GFX8-NEXT: s_lshl_b32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s2
+; GFX8-NEXT: v_lshrrev_b16_e32 v0, s1, v0
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i16_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_lshrrev_b16 v0.l, 1, v0.l
+; GFX11-NEXT: s_and_not1_b32 s2, 15, s1
+; GFX11-NEXT: s_and_b32 s1, s1, 15
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i16_svs:
; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s2, s1, 15
+; GFX9-NEXT: s_andn2_b32 s1, 15, s1
; GFX9-NEXT: v_lshrrev_b16_e32 v0, 1, v0
-; GFX9-NEXT: s_andn2_b32 s2, 15, s1
-; GFX9-NEXT: s_and_b32 s1, s1, 15
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, s2, v0
-; GFX9-NEXT: s_lshl_b32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s2
+; GFX9-NEXT: v_lshrrev_b16_e32 v0, s1, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshrrev_b16 v0, 1, v0
@@ -3566,19 +3693,6 @@ define amdgpu_ps half @v_fshl_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX10-NEXT: v_lshrrev_b16 v0, s2, v0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshl_i16_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshl_i16_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 1, v0
@@ -3598,69 +3712,66 @@ define amdgpu_ps half @v_fshl_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
define amdgpu_ps half @v_fshl_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt) {
; GFX6-LABEL: v_fshl_i16_vss:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_and_b32 s2, s1, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: s_bfe_u32 s0, s0, 0xf0001
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX6-NEXT: s_lshr_b32 s0, s0, s1
; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX6-NEXT: s_and_b32 s0, s1, 15
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i16_vss:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
+; GFX8-NEXT: s_and_b32 s2, s1, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
; GFX8-NEXT: s_lshr_b32 s0, s0, 1
-; GFX8-NEXT: s_and_b32 s1, s1, 15
-; GFX8-NEXT: s_lshr_b32 s0, s0, s2
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, s1, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, s2, v0
+; GFX8-NEXT: s_lshr_b32 s0, s0, s1
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i16_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s2, s1, 15
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
+; GFX11-NEXT: s_and_not1_b32 s1, 15, s1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, s0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i16_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX9-NEXT: s_andn2_b32 s2, 15, s1
+; GFX9-NEXT: s_and_b32 s2, s1, 15
+; GFX9-NEXT: s_andn2_b32 s1, 15, s1
; GFX9-NEXT: s_lshr_b32 s0, s0, 1
-; GFX9-NEXT: s_and_b32 s1, s1, 15
-; GFX9-NEXT: s_lshr_b32 s0, s0, s2
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, s1, v0
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, s2, v0
+; GFX9-NEXT: s_lshr_b32 s0, s0, s1
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i16_vss:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_andn2_b32 s2, 15, s1
-; GFX10-NEXT: s_and_b32 s1, s1, 15
+; GFX10-NEXT: s_and_b32 s2, s1, 15
; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX10-NEXT: v_lshlrev_b16 v0, s1, v0
+; GFX10-NEXT: v_lshlrev_b16 v0, s2, v0
+; GFX10-NEXT: s_andn2_b32 s1, 15, s1
; GFX10-NEXT: s_lshr_b32 s0, s0, 1
-; GFX10-NEXT: s_lshr_b32 s0, s0, s2
+; GFX10-NEXT: s_lshr_b32 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshl_i16_vss:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, 15
-; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s1, 15, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshl_i16_vss:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 15
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s1, 15
; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, s1, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, s2, v0
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s1, 15, s1
; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
@@ -3707,6 +3818,29 @@ define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s6, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_and_b32 s3, s2, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s6, s6, 0x10001
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshl_b32 s3, s4, s5
+; GFX11-NEXT: s_and_b32 s4, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_lshr_b32 s5, s2, 16
+; GFX11-NEXT: s_lshr_b32 s2, s4, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s3, s2, 0xf000f
@@ -3729,7 +3863,6 @@ define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s6, s1, 0xffff
@@ -3752,30 +3885,6 @@ define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_and_b32 s3, s2, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s6, s6, 0x10001
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
-; GFX11-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-NEXT: s_lshr_b32 s5, s3, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshl_b32 s3, s4, s5
-; GFX11-NEXT: s_and_b32 s4, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_lshr_b32 s5, s2, 16
-; GFX11-NEXT: s_lshr_b32 s2, s4, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to i32
ret i32 %cast
@@ -3815,6 +3924,19 @@ define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, v3, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3826,7 +3948,6 @@ define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3838,20 +3959,6 @@ define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX10-NEXT: v_pk_lshrrev_b16 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
-; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, 1, v1 op_sel_hi:[0,1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, v3, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
ret <2 x i16> %result
}
@@ -3881,6 +3988,14 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v2i16_4_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x80004, v0
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x8000c, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i16_4_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3890,7 +4005,6 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v2i16_4_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3898,15 +4012,6 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX10-NEXT: v_pk_lshrrev_b16 v1, 0x8000c, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_v2i16_4_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x80004, v0
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x8000c, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> <i16 4, i16 8>)
ret <2 x i16> %result
}
@@ -3946,6 +4051,22 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_v2i16_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX11-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
+; GFX11-NEXT: s_lshr_b32 s2, s2, 0x10001
+; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v0, s0
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_v2i16_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_and_b32_e32 v1, 0xf000f, v0
@@ -3960,7 +4081,6 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX9-NEXT: v_pk_lshrrev_b16 v0, v0, s0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_v2i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -3975,23 +4095,6 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX10-NEXT: v_pk_lshrrev_b16 v1, v1, s1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshl_v2i16_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX11-NEXT: s_and_b32 s2, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX11-NEXT: s_lshr_b32 s2, s2, 0x10001
-; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v0, s0
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, v1, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4033,6 +4136,21 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_v2i16_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, s1, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s1, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_v2i16_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
@@ -4046,7 +4164,6 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX9-NEXT: v_pk_lshrrev_b16 v0, s1, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_v2i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
@@ -4060,22 +4177,6 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshl_v2i16_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshl_b32 s1, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4117,6 +4218,25 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_v2i16_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s3, 0x10001
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, s2, v0
+; GFX11-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_lshr_b32 s1, s2, s1
+; GFX11-NEXT: s_lshr_b32 s0, s0, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_v2i16_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
@@ -4135,7 +4255,6 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_v2i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s3, s0, 0xffff
@@ -4154,26 +4273,6 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshl_v2i16_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_lshr_b32 s3, s3, 0x10001
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s3, s0
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, s2, v0
-; GFX11-NEXT: s_and_b32 s2, s0, 0xffff
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-NEXT: s_lshr_b32 s1, s2, s1
-; GFX11-NEXT: s_lshr_b32 s0, s0, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4247,148 +4346,184 @@ define amdgpu_ps i48 @s_fshl_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX11-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-NEXT: s_lshr_b32 s8, s6, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_lshl_b32 s6, s7, s8
+; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-NEXT: s_lshr_b32 s4, s7, s4
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_and_b32 s8, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s4, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s8, s8, 0x10001
+; GFX11-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s5, 0xf000f, s5
+; GFX11-NEXT: s_lshr_b32 s6, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s8, s3
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s7
+; GFX11-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-NEXT: s_lshr_b32 s5, s6, s5
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_andn2_b32 s6, 15, s5
-; GFX9-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX9-NEXT: s_lshr_b32 s6, s7, s6
-; GFX9-NEXT: s_and_b32 s7, s5, 15
-; GFX9-NEXT: s_lshl_b32 s7, s1, s7
-; GFX9-NEXT: s_or_b32 s6, s7, s6
-; GFX9-NEXT: s_lshr_b32 s7, s5, 16
-; GFX9-NEXT: s_lshr_b32 s3, s3, 17
-; GFX9-NEXT: s_andn2_b32 s7, 15, s7
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX9-NEXT: s_lshr_b32 s3, s3, s7
-; GFX9-NEXT: s_lshl_b32 s1, s1, s5
-; GFX9-NEXT: s_or_b32 s1, s1, s3
-; GFX9-NEXT: s_andn2_b32 s3, 15, s4
-; GFX9-NEXT: s_bfe_u32 s5, s2, 0xf0001
-; GFX9-NEXT: s_lshr_b32 s3, s5, s3
-; GFX9-NEXT: s_and_b32 s5, s4, 15
-; GFX9-NEXT: s_lshl_b32 s5, s0, s5
-; GFX9-NEXT: s_or_b32 s3, s5, s3
+; GFX9-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_lshr_b32 s8, s6, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s6
+; GFX9-NEXT: s_lshl_b32 s6, s7, s8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX9-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s6, s6, 0x10001
+; GFX9-NEXT: s_lshr_b32 s2, s2, 1
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX9-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s7, s4, 16
+; GFX9-NEXT: s_lshr_b32 s4, s6, s4
+; GFX9-NEXT: s_lshr_b32 s2, s2, s7
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s5, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s5
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
+; GFX9-NEXT: s_lshr_b32 s6, s2, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, s2
+; GFX9-NEXT: s_lshl_b32 s2, s5, s6
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: s_lshr_b32 s2, s2, 0x10001
+; GFX9-NEXT: s_lshr_b32 s3, s3, 1
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT: s_and_b32 s3, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
; GFX9-NEXT: s_lshr_b32 s5, s4, 16
-; GFX9-NEXT: s_lshr_b32 s2, s2, 17
-; GFX9-NEXT: s_andn2_b32 s5, 15, s5
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX9-NEXT: s_lshr_b32 s3, s3, s4
; GFX9-NEXT: s_lshr_b32 s2, s2, s5
-; GFX9-NEXT: s_lshl_b32 s0, s0, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
; GFX9-NEXT: s_or_b32 s0, s0, s2
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_v3i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_andn2_b32 s6, 15, s5
-; GFX10-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX10-NEXT: s_and_b32 s8, s5, 15
-; GFX10-NEXT: s_lshr_b32 s6, s7, s6
-; GFX10-NEXT: s_lshl_b32 s7, s1, s8
-; GFX10-NEXT: s_lshr_b32 s3, s3, 17
-; GFX10-NEXT: s_or_b32 s6, s7, s6
-; GFX10-NEXT: s_lshr_b32 s7, s5, 16
-; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: s_andn2_b32 s7, 15, s7
-; GFX10-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX10-NEXT: s_lshr_b32 s3, s3, s7
-; GFX10-NEXT: s_lshl_b32 s1, s1, s5
-; GFX10-NEXT: s_andn2_b32 s5, 15, s4
-; GFX10-NEXT: s_bfe_u32 s7, s2, 0xf0001
-; GFX10-NEXT: s_and_b32 s8, s4, 15
-; GFX10-NEXT: s_lshr_b32 s5, s7, s5
-; GFX10-NEXT: s_lshl_b32 s7, s0, s8
+; GFX10-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX10-NEXT: s_lshr_b32 s2, s2, 16
+; GFX10-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX10-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX10-NEXT: s_lshr_b32 s2, s2, 1
+; GFX10-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX10-NEXT: s_lshr_b32 s7, s0, 16
+; GFX10-NEXT: s_lshr_b32 s8, s6, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX10-NEXT: s_lshl_b32 s0, s0, s6
+; GFX10-NEXT: s_lshl_b32 s6, s7, s8
+; GFX10-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX10-NEXT: s_lshr_b32 s2, s2, 16
; GFX10-NEXT: s_lshr_b32 s8, s4, 16
-; GFX10-NEXT: s_lshr_b32 s2, s2, 17
-; GFX10-NEXT: s_andn2_b32 s8, 15, s8
-; GFX10-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX10-NEXT: s_lshr_b32 s4, s7, s4
; GFX10-NEXT: s_lshr_b32 s2, s2, s8
-; GFX10-NEXT: s_lshl_b32 s0, s0, s4
-; GFX10-NEXT: s_or_b32 s4, s7, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX10-NEXT: s_and_b32 s7, s3, 0xffff
+; GFX10-NEXT: s_lshr_b32 s3, s3, 16
; GFX10-NEXT: s_or_b32 s0, s0, s2
-; GFX10-NEXT: s_or_b32 s1, s1, s3
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX10-NEXT: s_and_b32 s2, s5, 0xf000f
+; GFX10-NEXT: s_lshr_b32 s7, s7, 0x10001
+; GFX10-NEXT: s_lshr_b32 s3, s3, 1
+; GFX10-NEXT: s_andn2_b32 s4, 0xf000f, s5
+; GFX10-NEXT: s_lshr_b32 s5, s1, 16
+; GFX10-NEXT: s_lshr_b32 s6, s2, 16
+; GFX10-NEXT: s_lshl_b32 s1, s1, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s7, s3
+; GFX10-NEXT: s_lshl_b32 s3, s5, s6
+; GFX10-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX10-NEXT: s_lshr_b32 s2, s2, 16
+; GFX10-NEXT: s_lshr_b32 s6, s4, 16
+; GFX10-NEXT: s_lshr_b32 s4, s5, s4
+; GFX10-NEXT: s_lshr_b32 s2, s2, s6
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s3, s3, 16
+; GFX10-NEXT: s_or_b32 s1, s1, s2
+; GFX10-NEXT: s_or_b32 s0, s0, s3
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_fshl_v3i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s4, 0xf000f
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s9, 0x10001
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 1
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s6, 16
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s9, s2
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s6
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s7, s8
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s7, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s3, 0xffff
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s4, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, 0xf000f
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s8, 0x10001
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 1
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s5, 0xf000f, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s8, s3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s7
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s3, 0xffff
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
-; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: s_fshl_v3i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s6, 15, s5
-; GFX11-FAKE16-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX11-FAKE16-NEXT: s_and_b32 s8, s5, 15
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s7, s6
-; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s1, s8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 17
-; GFX11-FAKE16-NEXT: s_or_b32 s6, s7, s6
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s7, 15, s7
-; GFX11-FAKE16-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s5
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s5, 15, s4
-; GFX11-FAKE16-NEXT: s_bfe_u32 s7, s2, 0xf0001
-; GFX11-FAKE16-NEXT: s_and_b32 s8, s4, 15
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s7, s5
-; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s0, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s6, 16
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s7, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 17
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-FAKE16-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s7, s4
; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-FAKE16-NEXT: s_or_b32 s4, s7, s5
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s3, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s3
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s5, 0xf000f
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s7, 0x10001
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s4, 0xf000f, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s7, s3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s5, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s5, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s6
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call <3 x i16> @llvm.fshl.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
%cast = bitcast <3 x i16> %result to i48
@@ -4440,98 +4575,62 @@ define <3 x half> @v_fshl_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v4, v0
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v5, v1
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v7, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v7, -1
-; GFX9-NEXT: v_xor_b32_sdwa v7, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 17, v2
-; GFX9-NEXT: v_lshlrev_b16_sdwa v6, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b16_e32 v7, v7, v8
-; GFX9-NEXT: v_or_b32_e32 v6, v6, v7
-; GFX9-NEXT: v_lshrrev_b16_e32 v3, 1, v3
-; GFX9-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX9-NEXT: v_lshrrev_b16_e32 v3, v7, v3
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, v5, v1
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_lshrrev_b16_e32 v2, 1, v2
-; GFX9-NEXT: v_xor_b32_e32 v3, -1, v4
-; GFX9-NEXT: v_lshrrev_b16_e32 v2, v3, v2
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, v4, v0
+; GFX9-NEXT: v_and_b32_e32 v6, 0xf000f, v4
+; GFX9-NEXT: v_xor_b32_e32 v4, -1, v4
+; GFX9-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX9-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, v6, v0
+; GFX9-NEXT: v_pk_lshrrev_b16 v2, v4, v2
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v6, v0, s4
+; GFX9-NEXT: v_and_b32_e32 v2, 0xf000f, v5
+; GFX9-NEXT: v_xor_b32_e32 v4, -1, v5
+; GFX9-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, v2, v1
+; GFX9-NEXT: v_pk_lshrrev_b16 v2, 1, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshrrev_b16 v2, v4, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v3i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 17, v2
-; GFX10-NEXT: v_lshrrev_b16 v2, 1, v2
-; GFX10-NEXT: v_xor_b32_e32 v9, -1, v4
-; GFX10-NEXT: v_xor_b32_e32 v10, -1, v6
-; GFX10-NEXT: v_lshlrev_b16 v0, v4, v0
-; GFX10-NEXT: v_lshlrev_b16 v4, v6, v7
-; GFX10-NEXT: v_lshrrev_b16 v3, 1, v3
-; GFX10-NEXT: v_lshrrev_b16 v2, v9, v2
-; GFX10-NEXT: v_lshrrev_b16 v6, v10, v8
+; GFX10-NEXT: v_xor_b32_e32 v6, -1, v4
; GFX10-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX10-NEXT: v_lshlrev_b16 v1, v5, v1
+; GFX10-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX10-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX10-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX10-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, v4, v0
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, v5, v1
+; GFX10-NEXT: v_pk_lshrrev_b16 v2, v6, v2
+; GFX10-NEXT: v_pk_lshrrev_b16 v3, v7, v3
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v2, v4, v6
-; GFX10-NEXT: v_lshrrev_b16 v3, v7, v3
-; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshl_v3i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, v4, v0
-; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, v5, v1
-; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v2, v6, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v3, v7, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_fshl_v3i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 17, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, 1, v2
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v9, -1, v4
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v10, -1, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v4, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, v6, v7
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 1, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, v9, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v6, v10, v8
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, v5, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v4, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, v7, v3
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <3 x i16> @llvm.fshl.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
%cast.result = bitcast <3 x i16> %result to <3 x half>
ret <3 x half> %cast.result
@@ -4604,6 +4703,47 @@ define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX11-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-NEXT: s_lshr_b32 s8, s6, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_lshl_b32 s6, s7, s8
+; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-NEXT: s_lshr_b32 s4, s7, s4
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_and_b32 s8, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s4, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s8, s8, 0x10001
+; GFX11-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s5, 0xf000f, s5
+; GFX11-NEXT: s_lshr_b32 s6, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s8, s3
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s7
+; GFX11-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-NEXT: s_lshr_b32 s5, s6, s5
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s6, s4, 0xf000f
@@ -4645,7 +4785,6 @@ define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s2
; GFX9-NEXT: s_or_b32 s1, s1, s2
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s9, s2, 0xffff
@@ -4687,48 +4826,6 @@ define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_and_b32 s6, s4, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s9, s9, 0x10001
-; GFX11-NEXT: s_lshr_b32 s2, s2, 1
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s6, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s9, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, s6
-; GFX11-NEXT: s_lshl_b32 s6, s7, s8
-; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshr_b32 s4, s7, s4
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_and_b32 s8, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s2
-; GFX11-NEXT: s_and_b32 s4, s5, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s8, s8, 0x10001
-; GFX11-NEXT: s_lshr_b32 s3, s3, 1
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_and_not1_b32 s5, 0xf000f, s5
-; GFX11-NEXT: s_lshr_b32 s6, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s8, s3
-; GFX11-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-NEXT: s_lshl_b32 s4, s6, s7
-; GFX11-NEXT: s_and_b32 s6, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-NEXT: s_lshr_b32 s5, s6, s5
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
%result = call <4 x i16> @llvm.fshl.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <2 x i32>
ret <2 x i32> %cast.result
@@ -4790,6 +4887,26 @@ define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX8-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v4, v0
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v5, v1
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v7, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4808,7 +4925,6 @@ define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX9-NEXT: v_pk_lshrrev_b16 v2, v4, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4827,51 +4943,30 @@ define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v4, v0
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v5, v1
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, v7, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x i16> @llvm.fshl.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <4 x half>
ret <4 x half> %cast.result
}
define amdgpu_ps i64 @s_fshl_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt) {
-; GFX6-LABEL: s_fshl_i64:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX6-NEXT: s_not_b32 s4, s4
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshl_i64:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX8-NEXT: s_not_b32 s4, s4
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshl_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
+; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GCN-NEXT: s_not_b32 s4, s4
+; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_not_b32 s5, s4
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
@@ -4880,7 +4975,6 @@ define amdgpu_ps i64 @s_fshl_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
@@ -4889,16 +4983,6 @@ define amdgpu_ps i64 @s_fshl_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_not_b32 s5, s4
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -4992,6 +5076,19 @@ define i64 @v_fshl_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
+; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v5, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v4, v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5003,7 +5100,6 @@ define i64 @v_fshl_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5015,20 +5111,6 @@ define i64 @v_fshl_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
-; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
-; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v5, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v4, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -5050,22 +5132,6 @@ define i64 @v_fshl_i64_5(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_fshl_i64_5:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], 5, v[0:1]
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 27, v3
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_fshl_i64_5:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], 5, v[0:1]
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 27, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_fshl_i64_5:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5074,6 +5140,20 @@ define i64 @v_fshl_i64_5(i64 %lhs, i64 %rhs) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: v_fshl_i64_5:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], 5, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 27, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_fshl_i64_5:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], 5, v[0:1]
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 27, v3
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 5)
ret i64 %result
}
@@ -5114,6 +5194,14 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i64_48:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v4, v0
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i64_48:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5121,7 +5209,6 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i64_48:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5129,15 +5216,6 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_i64_48:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v1, v4, 16, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 48)
ret i64 %result
}
@@ -5165,6 +5243,18 @@ define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX8-NEXT: v_or_b32_e32 v1, v2, v4
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i64_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_and_b32_e32 v1, 63, v0
+; GFX11-NEXT: v_bfi_b32 v2, v0, 0, 63
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i64_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_and_b32_e32 v1, 63, v0
@@ -5175,7 +5265,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX9-NEXT: v_or_b32_e32 v0, v1, v3
; GFX9-NEXT: v_or_b32_e32 v1, v2, v4
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i64_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_and_b32_e32 v1, 63, v0
@@ -5186,19 +5275,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshl_i64_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_and_b32_e32 v1, 63, v0
-; GFX11-NEXT: v_bfi_b32 v2, v0, 0, 63
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -5229,6 +5305,18 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i64_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i64_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
@@ -5240,7 +5328,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i64_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
@@ -5252,19 +5339,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshl_i64_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -5297,6 +5371,18 @@ define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i64_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 63
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i64_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s3, s2, 63
@@ -5309,7 +5395,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i64_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
@@ -5322,53 +5407,39 @@ define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshl_i64_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s3, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 63
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
}
define amdgpu_ps <2 x i64> @s_fshl_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs, <2 x i64> inreg %amt) {
-; GFX6-LABEL: s_fshl_v2i64:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX6-NEXT: s_not_b32 s8, s8
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
-; GFX6-NEXT: s_not_b32 s6, s10
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshl_v2i64:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX8-NEXT: s_not_b32 s8, s8
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
-; GFX8-NEXT: s_not_b32 s6, s10
-; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshl_v2i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
+; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GCN-NEXT: s_not_b32 s8, s8
+; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GCN-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GCN-NEXT: s_not_b32 s6, s10
+; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
+; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshl_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX11-NEXT: s_not_b32 s9, s8
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX11-NEXT: s_not_b32 s8, s10
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s9
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s8
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
@@ -5382,7 +5453,6 @@ define amdgpu_ps <2 x i64> @s_fshl_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshl_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
@@ -5396,20 +5466,6 @@ define amdgpu_ps <2 x i64> @s_fshl_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshl_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX11-NEXT: s_not_b32 s9, s8
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX11-NEXT: s_not_b32 s8, s10
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s9
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s8
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5453,6 +5509,28 @@ define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], 1, v[6:7]
+; GFX11-NEXT: v_and_b32_e32 v9, 63, v8
+; GFX11-NEXT: v_bfi_b32 v8, v8, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v11, 63, v10
+; GFX11-NEXT: v_bfi_b32 v10, v10, 0, 63
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5471,7 +5549,6 @@ define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5490,29 +5567,6 @@ define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], 1, v[6:7]
-; GFX11-NEXT: v_and_b32_e32 v9, 63, v8
-; GFX11-NEXT: v_bfi_b32 v8, v8, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v11, 63, v10
-; GFX11-NEXT: v_bfi_b32 v10, v10, 0, 63
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5565,50 +5619,6 @@ define amdgpu_ps i128 @s_fshl_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshl_i128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s10, s8, 64
-; GFX9-NEXT: s_mov_b32 s11, 0
-; GFX9-NEXT: s_cmp_lg_u64 s[10:11], 0
-; GFX9-NEXT: s_cselect_b32 s11, s7, s1
-; GFX9-NEXT: s_cselect_b32 s10, s6, s0
-; GFX9-NEXT: s_cselect_b32 s1, s1, s3
-; GFX9-NEXT: s_cselect_b32 s0, s0, s2
-; GFX9-NEXT: s_cselect_b32 s5, s5, s7
-; GFX9-NEXT: s_cselect_b32 s4, s4, s6
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[10:11], 1
-; GFX9-NEXT: s_not_b32 s6, s8
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX9-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[10:11], s8
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshl_i128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s10, s8, 64
-; GFX10-NEXT: s_mov_b32 s11, 0
-; GFX10-NEXT: s_cmp_lg_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s11, s7, s1
-; GFX10-NEXT: s_cselect_b32 s10, s6, s0
-; GFX10-NEXT: s_cselect_b32 s1, s1, s3
-; GFX10-NEXT: s_cselect_b32 s0, s0, s2
-; GFX10-NEXT: s_cselect_b32 s3, s5, s7
-; GFX10-NEXT: s_cselect_b32 s2, s4, s6
-; GFX10-NEXT: s_lshl_b64 s[4:5], s[0:1], s8
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[10:11], 1
-; GFX10-NEXT: s_not_b32 s9, s8
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX10-NEXT: s_lshl_b64 s[6:7], s[10:11], s8
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s9
-; GFX10-NEXT: s_lshr_b64 s[8:9], s[0:1], s9
-; GFX10-NEXT: s_or_b64 s[0:1], s[6:7], s[2:3]
-; GFX10-NEXT: s_or_b64 s[2:3], s[4:5], s[8:9]
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: s_fshl_i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s10, s8, 64
@@ -5631,6 +5641,98 @@ define amdgpu_ps i128 @s_fshl_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX11-NEXT: s_or_b64 s[0:1], s[6:7], s[2:3]
; GFX11-NEXT: s_or_b64 s[2:3], s[4:5], s[8:9]
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshl_i128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX9-NEXT: s_sub_i32 s11, s9, 64
+; GFX9-NEXT: s_sub_i32 s14, 64, s9
+; GFX9-NEXT: s_cmp_lt_u32 s9, 64
+; GFX9-NEXT: s_cselect_b32 s18, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s9, 0
+; GFX9-NEXT: s_cselect_b32 s9, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[14:15], s[0:1], s14
+; GFX9-NEXT: s_lshl_b64 s[16:17], s[2:3], s8
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[0:1], s8
+; GFX9-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s11
+; GFX9-NEXT: s_cmp_lg_u32 s18, 0
+; GFX9-NEXT: s_cselect_b64 s[12:13], s[12:13], 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[14:15], s[0:1]
+; GFX9-NEXT: s_cmp_lg_u32 s9, 0
+; GFX9-NEXT: s_mov_b32 s10, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX9-NEXT: s_lshl_b32 s11, s6, 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GFX9-NEXT: s_andn2_b32 s6, 0x7f, s8
+; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX9-NEXT: s_not_b32 s9, s8
+; GFX9-NEXT: s_sub_i32 s14, s6, 64
+; GFX9-NEXT: s_sub_i32 s10, 64, s6
+; GFX9-NEXT: s_cmp_lt_u32 s6, 64
+; GFX9-NEXT: s_cselect_b32 s15, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s6, 0
+; GFX9-NEXT: s_cselect_b32 s16, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[4:5], s9
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[0:1], s9
+; GFX9-NEXT: s_lshl_b64 s[10:11], s[4:5], s10
+; GFX9-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s14
+; GFX9-NEXT: s_cmp_lg_u32 s15, 0
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX9-NEXT: s_cmp_lg_u32 s16, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT: s_cmp_lg_u32 s15, 0
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[6:7], 0
+; GFX9-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
+; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshl_i128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX10-NEXT: s_mov_b32 s10, 0
+; GFX10-NEXT: s_sub_i32 s11, s9, 64
+; GFX10-NEXT: s_sub_i32 s12, 64, s9
+; GFX10-NEXT: s_cmp_lt_u32 s9, 64
+; GFX10-NEXT: s_cselect_b32 s18, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[12:13], s[0:1], s12
+; GFX10-NEXT: s_lshl_b64 s[14:15], s[2:3], s8
+; GFX10-NEXT: s_lshl_b64 s[16:17], s[0:1], s8
+; GFX10-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s11
+; GFX10-NEXT: s_cmp_lg_u32 s18, 0
+; GFX10-NEXT: s_cselect_b64 s[14:15], s[16:17], 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX10-NEXT: s_lshl_b32 s11, s6, 31
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GFX10-NEXT: s_andn2_b32 s6, 0x7f, s8
+; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT: s_not_b32 s10, s8
+; GFX10-NEXT: s_sub_i32 s12, s6, 64
+; GFX10-NEXT: s_sub_i32 s8, 64, s6
+; GFX10-NEXT: s_cmp_lt_u32 s6, 64
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s6, 0
+; GFX10-NEXT: s_cselect_b32 s16, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[0:1], s10
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[4:5], s8
+; GFX10-NEXT: s_lshr_b64 s[10:11], s[4:5], s10
+; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s12
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT: s_cmp_lg_u32 s16, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[10:11], 0
+; GFX10-NEXT: s_or_b64 s[0:1], s[14:15], s[0:1]
+; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -5734,6 +5836,54 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX8-NEXT: v_or_b32_e32 v3, v13, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v17, 0x7f, v8
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX11-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
+; GFX11-NEXT: v_lshrrev_b64 v[9:10], 1, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v11, 64, v17
+; GFX11-NEXT: v_lshlrev_b64 v[7:8], v17, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v15, 0xffffffc0, v17
+; GFX11-NEXT: v_lshl_or_b32 v5, v6, 31, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v18
+; GFX11-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[13:14], v17, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v17
+; GFX11-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v18
+; GFX11-NEXT: v_lshlrev_b64 v[15:16], v16, v[9:10]
+; GFX11-NEXT: v_or_b32_e32 v11, v11, v7
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], v18, v[4:5]
+; GFX11-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v18
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v0, v11, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[11:12], v19, v[9:10]
+; GFX11-NEXT: v_or_b32_e32 v0, v6, v15
+; GFX11-NEXT: v_or_b32_e32 v6, v7, v16
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v8, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v17
+; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v11, v0, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v18, v[9:10]
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, v6, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v9, 0, v14, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v20, v2, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, v4, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v5, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v7, 0, v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v13, v4
+; GFX11-NEXT: v_or_b32_e32 v1, v9, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5780,7 +5930,6 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX9-NEXT: v_or_b32_e32 v2, v10, v2
; GFX9-NEXT: v_or_b32_e32 v3, v13, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5826,55 +5975,6 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v17, 0x7f, v8
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
-; GFX11-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
-; GFX11-NEXT: v_lshrrev_b64 v[9:10], 1, v[6:7]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_sub_nc_u32_e32 v11, 64, v17
-; GFX11-NEXT: v_lshlrev_b64 v[7:8], v17, v[2:3]
-; GFX11-NEXT: v_add_nc_u32_e32 v15, 0xffffffc0, v17
-; GFX11-NEXT: v_lshl_or_b32 v5, v6, 31, v5
-; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v18
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[13:14], v17, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
-; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v17
-; GFX11-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v18
-; GFX11-NEXT: v_lshlrev_b64 v[15:16], v16, v[9:10]
-; GFX11-NEXT: v_or_b32_e32 v11, v11, v7
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v18, v[4:5]
-; GFX11-NEXT: v_or_b32_e32 v8, v12, v8
-; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v18
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v18
-; GFX11-NEXT: v_cndmask_b32_e32 v20, v0, v11, vcc_lo
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], v19, v[9:10]
-; GFX11-NEXT: v_or_b32_e32 v0, v6, v15
-; GFX11-NEXT: v_or_b32_e32 v6, v7, v16
-; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v8, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v17
-; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v11, v0, s1
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v18, v[9:10]
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, v6, s1
-; GFX11-NEXT: v_cndmask_b32_e32 v9, 0, v14, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v20, v2, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, v4, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v5, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v7, 0, v1, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v13, v4
-; GFX11-NEXT: v_or_b32_e32 v1, v9, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -5944,68 +6044,6 @@ define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: v_fshl_i128_ssv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[1:2]
-; GFX9-NEXT: v_mov_b32_e32 v8, s1
-; GFX9-NEXT: v_mov_b32_e32 v6, s7
-; GFX9-NEXT: v_mov_b32_e32 v7, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v6, v7, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v6, vcc
-; GFX9-NEXT: v_mov_b32_e32 v9, s0
-; GFX9-NEXT: v_lshrrev_b64 v[6:7], 1, v[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v1, vcc
-; GFX9-NEXT: v_not_b32_e32 v10, v0
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], v0, v[2:3]
-; GFX9-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
-; GFX9-NEXT: v_or_b32_e32 v1, v5, v7
-; GFX9-NEXT: v_mov_b32_e32 v5, s3
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v5, v8, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[7:8], v0, v[7:8]
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], v10, v[2:3]
-; GFX9-NEXT: v_or_b32_e32 v0, v4, v6
-; GFX9-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX9-NEXT: v_or_b32_e32 v2, v7, v2
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: v_fshl_i128_ssv:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_mov_b32_e32 v3, s7
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s0
-; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[1:2]
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: v_not_b32_e32 v9, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s1, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s7, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s6, v5, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s0, v6, vcc_lo
-; GFX10-NEXT: v_lshrrev_b64 v[3:4], 1, v[3:4]
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s3, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s2, v7, vcc_lo
-; GFX10-NEXT: v_lshrrev_b64 v[7:8], 1, v[1:2]
-; GFX10-NEXT: v_lshlrev_b64 v[1:2], v0, v[1:2]
-; GFX10-NEXT: v_lshrrev_b64 v[3:4], v9, v[3:4]
-; GFX10-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
-; GFX10-NEXT: v_lshrrev_b64 v[7:8], v9, v[7:8]
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v3
-; GFX10-NEXT: v_or_b32_e32 v1, v2, v4
-; GFX10-NEXT: v_or_b32_e32 v2, v5, v7
-; GFX10-NEXT: v_or_b32_e32 v3, v6, v8
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: v_fshl_i128_ssv:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 64, v0
@@ -6039,6 +6077,105 @@ define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v6, v8
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: v_fshl_i128_ssv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_and_b32_e32 v8, 0x7f, v0
+; GFX9-NEXT: v_sub_u32_e32 v1, 64, v8
+; GFX9-NEXT: v_lshrrev_b64 v[1:2], v1, s[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[3:4], v8, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v9, 0xffffffc0, v8
+; GFX9-NEXT: v_lshlrev_b64 v[5:6], v8, s[0:1]
+; GFX9-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX9-NEXT: v_lshlrev_b64 v[1:2], v9, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7f
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8
+; GFX9-NEXT: s_mov_b32 s8, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX9-NEXT: s_lshl_b32 s9, s6, 31
+; GFX9-NEXT: v_bfi_b32 v12, v0, 0, v7
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v2, v4, vcc
+; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[6:7], 1
+; GFX9-NEXT: v_sub_u32_e32 v4, 64, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v1, v3, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], v12, s[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v4, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v13, 0xffffffc0, v12
+; GFX9-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX9-NEXT: v_or_b32_e32 v5, v3, v5
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], v13, s[2:3]
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v12, s[2:3]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
+; GFX9-NEXT: v_mov_b32_e32 v7, s1
+; GFX9-NEXT: v_mov_b32_e32 v6, s0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v12
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, v9, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v10, v3
+; GFX9-NEXT: v_or_b32_e32 v2, v8, v4
+; GFX9-NEXT: v_or_b32_e32 v3, v11, v5
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: v_fshl_i128_ssv:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_and_b32_e32 v11, 0x7f, v0
+; GFX10-NEXT: v_bfi_b32 v12, v0, 0, 0x7f
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX10-NEXT: s_lshl_b32 s9, s6, 31
+; GFX10-NEXT: v_sub_nc_u32_e32 v3, 64, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffc0, v11
+; GFX10-NEXT: v_lshlrev_b64 v[1:2], v11, s[2:3]
+; GFX10-NEXT: v_sub_nc_u32_e32 v9, 64, v12
+; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX10-NEXT: v_lshrrev_b64 v[3:4], v3, s[0:1]
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX10-NEXT: v_lshlrev_b64 v[7:8], v7, s[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[5:6], v11, s[0:1]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v13, 0xffffffc0, v12
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v1
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v12, s[4:5]
+; GFX10-NEXT: v_lshlrev_b64 v[9:10], v9, s[6:7]
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v14, 0, v5, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], v13, s[6:7]
+; GFX10-NEXT: v_cmp_gt_u32_e64 s1, 64, v12
+; GFX10-NEXT: v_or_b32_e32 v5, v0, v9
+; GFX10-NEXT: v_or_b32_e32 v9, v1, v10
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v4, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v11
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v5, s1
+; GFX10-NEXT: v_lshrrev_b64 v[4:5], v12, s[6:7]
+; GFX10-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v12
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v9, s1
+; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, s2, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, s3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v0, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, v1, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v4, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, v5, s1
+; GFX10-NEXT: v_or_b32_e32 v0, v14, v0
+; GFX10-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX10-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX10-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -6161,6 +6298,61 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshl_i128_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: s_sub_i32 s12, s5, 64
+; GFX11-NEXT: s_sub_i32 s6, 64, s5
+; GFX11-NEXT: s_cmp_lt_u32 s5, 64
+; GFX11-NEXT: s_cselect_b32 s13, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: v_lshl_or_b32 v1, v2, 31, v1
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s4
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[0:1], s4
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-NEXT: s_and_not1_b32 s2, 0x7f, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s4, 64, s2
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX11-NEXT: s_sub_i32 s3, s2, 64
+; GFX11-NEXT: s_cmp_lt_u32 s2, 64
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i128_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s5, s4, 0x7f
@@ -6217,7 +6409,6 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshl_i128_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s5, s4, 0x7f
@@ -6274,62 +6465,6 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshl_i128_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s5, s4, 0x7f
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_sub_i32 s12, s5, 64
-; GFX11-NEXT: s_sub_i32 s6, 64, s5
-; GFX11-NEXT: s_cmp_lt_u32 s5, 64
-; GFX11-NEXT: s_cselect_b32 s13, 1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s5, 0
-; GFX11-NEXT: v_lshl_or_b32 v1, v2, 31, v1
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
-; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s4
-; GFX11-NEXT: s_lshl_b64 s[10:11], s[0:1], s4
-; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
-; GFX11-NEXT: s_cmp_lg_u32 s13, 0
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
-; GFX11-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
-; GFX11-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
-; GFX11-NEXT: s_cmp_lg_u32 s5, 0
-; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
-; GFX11-NEXT: s_and_not1_b32 s2, 0x7f, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s4, 64, s2
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
-; GFX11-NEXT: s_sub_i32 s3, s2, 64
-; GFX11-NEXT: s_cmp_lt_u32 s2, 64
-; GFX11-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 0
-; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
-; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
-; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
-; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -6393,61 +6528,6 @@ define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: v_fshl_i128_vss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s6, s4, 64
-; GFX9-NEXT: s_mov_b32 s7, 0
-; GFX9-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, s3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
-; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s1, s1, s3
-; GFX9-NEXT: s_cselect_b32 s0, s0, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], 1, v[5:6]
-; GFX9-NEXT: v_lshlrev_b64 v[7:8], s4, v[5:6]
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX9-NEXT: s_not_b32 s2, s4
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_lshlrev_b64 v[5:6], s4, v[0:1]
-; GFX9-NEXT: v_lshrrev_b64 v[1:2], s2, v[2:3]
-; GFX9-NEXT: v_or_b32_e32 v4, s1, v8
-; GFX9-NEXT: v_or_b32_e32 v0, s0, v7
-; GFX9-NEXT: v_or_b32_e32 v3, v6, v2
-; GFX9-NEXT: v_or_b32_e32 v2, v5, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: v_fshl_i128_vss:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s6, s4, 64
-; GFX10-NEXT: s_mov_b32 s7, 0
-; GFX10-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v1, s3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v0, s2, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT: s_and_b32 s5, vcc_lo, exec_lo
-; GFX10-NEXT: s_cselect_b32 s1, s1, s3
-; GFX10-NEXT: v_lshrrev_b64 v[2:3], 1, v[4:5]
-; GFX10-NEXT: s_cselect_b32 s0, s0, s2
-; GFX10-NEXT: s_not_b32 s2, s4
-; GFX10-NEXT: v_lshlrev_b64 v[4:5], s4, v[4:5]
-; GFX10-NEXT: v_lshlrev_b64 v[6:7], s4, v[0:1]
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: v_lshrrev_b64 v[2:3], s2, v[2:3]
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: v_or_b32_e32 v0, s0, v4
-; GFX10-NEXT: v_or_b32_e32 v1, s1, v5
-; GFX10-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX10-NEXT: v_or_b32_e32 v3, v7, v3
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: v_fshl_i128_vss:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s6, s4, 64
@@ -6476,63 +6556,142 @@ define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX11-NEXT: v_or_b32_e32 v2, v6, v2
; GFX11-NEXT: v_or_b32_e32 v3, v7, v3
; GFX11-NEXT: ; return to shader part epilog
- %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
- %cast.result = bitcast i128 %result to <4 x float>
- ret <4 x float> %cast.result
-}
-
-define amdgpu_ps i128 @s_fshl_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
-; GFX6-LABEL: s_fshl_i128_65:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
-; GFX6-NEXT: s_lshr_b32 s10, s7, 31
-; GFX6-NEXT: s_mov_b32 s11, 0
-; GFX6-NEXT: s_or_b64 s[8:9], s[2:3], s[10:11]
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
-; GFX6-NEXT: s_lshr_b32 s10, s5, 31
-; GFX6-NEXT: s_or_b64 s[6:7], s[0:1], s[10:11]
-; GFX6-NEXT: s_mov_b32 s0, s6
-; GFX6-NEXT: s_mov_b32 s2, s8
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshl_i128_65:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
-; GFX8-NEXT: s_lshr_b32 s10, s7, 31
-; GFX8-NEXT: s_mov_b32 s11, 0
-; GFX8-NEXT: s_or_b64 s[8:9], s[2:3], s[10:11]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
-; GFX8-NEXT: s_lshr_b32 s10, s5, 31
-; GFX8-NEXT: s_or_b64 s[6:7], s[0:1], s[10:11]
-; GFX8-NEXT: s_mov_b32 s0, s6
-; GFX8-NEXT: s_mov_b32 s2, s8
-; GFX8-NEXT: ; return to shader part epilog
-;
-; GFX9-LABEL: s_fshl_i128_65:
+; GFX9-LABEL: v_fshl_i128_vss:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
-; GFX9-NEXT: s_lshr_b32 s10, s7, 31
-; GFX9-NEXT: s_mov_b32 s11, 0
-; GFX9-NEXT: s_or_b64 s[8:9], s[2:3], s[10:11]
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
-; GFX9-NEXT: s_lshr_b32 s10, s5, 31
-; GFX9-NEXT: s_or_b64 s[6:7], s[0:1], s[10:11]
-; GFX9-NEXT: s_mov_b32 s0, s6
-; GFX9-NEXT: s_mov_b32 s2, s8
+; GFX9-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX9-NEXT: s_sub_i32 s7, s5, 64
+; GFX9-NEXT: s_sub_i32 s8, 64, s5
+; GFX9-NEXT: s_cmp_lt_u32 s5, 64
+; GFX9-NEXT: s_cselect_b32 s9, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s5, 0
+; GFX9-NEXT: s_cselect_b32 s10, 1, 0
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], s8, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
+; GFX9-NEXT: s_cmp_lg_u32 s9, 0
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_cmp_lg_u32 s10, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT: s_lshl_b32 s7, s2, 31
+; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX9-NEXT: s_not_b32 s6, s4
+; GFX9-NEXT: s_andn2_b32 s4, 0x7f, s4
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX9-NEXT: s_sub_i32 s10, s4, 64
+; GFX9-NEXT: s_sub_i32 s8, 64, s4
+; GFX9-NEXT: s_cmp_lt_u32 s4, 64
+; GFX9-NEXT: s_cselect_b32 s11, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-NEXT: s_cselect_b32 s12, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[2:3], s6
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX9-NEXT: s_cmp_lg_u32 s12, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX9-NEXT: v_or_b32_e32 v1, v7, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX9-NEXT: v_or_b32_e32 v3, v5, v3
; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshl_i128_65:
+; GFX10-LABEL: v_fshl_i128_vss:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
-; GFX10-NEXT: s_lshr_b32 s10, s7, 31
-; GFX10-NEXT: s_mov_b32 s11, 0
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
-; GFX10-NEXT: s_or_b64 s[8:9], s[2:3], s[10:11]
-; GFX10-NEXT: s_lshr_b32 s10, s5, 31
-; GFX10-NEXT: s_mov_b32 s2, s8
-; GFX10-NEXT: s_or_b64 s[6:7], s[0:1], s[10:11]
-; GFX10-NEXT: s_mov_b32 s0, s6
+; GFX10-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_sub_i32 s6, 64, s5
+; GFX10-NEXT: s_sub_i32 s7, s5, 64
+; GFX10-NEXT: s_cmp_lt_u32 s5, 64
+; GFX10-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX10-NEXT: s_cselect_b32 s6, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
+; GFX10-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX10-NEXT: s_cselect_b32 s6, exec_lo, 0
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT: s_lshl_b32 s9, s2, 31
+; GFX10-NEXT: s_andn2_b32 s7, 0x7f, s4
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_not_b32 s10, s4
+; GFX10-NEXT: s_sub_i32 s12, s7, 64
+; GFX10-NEXT: s_sub_i32 s8, 64, s7
+; GFX10-NEXT: s_cmp_lt_u32 s7, 64
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s7, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
+; GFX10-NEXT: s_cselect_b32 s7, 1, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[0:1], s10
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX10-NEXT: s_lshr_b64 s[10:11], s[2:3], s10
+; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX10-NEXT: s_cmp_lg_u32 s7, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[10:11], 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
+; GFX10-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX10-NEXT: v_or_b32_e32 v1, v7, v1
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v5
; GFX10-NEXT: ; return to shader part epilog
+ %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
+ %cast.result = bitcast i128 %result to <4 x float>
+ ret <4 x float> %cast.result
+}
+
+define amdgpu_ps i128 @s_fshl_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
+; GCN-LABEL: s_fshl_i128_65:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
+; GCN-NEXT: s_lshr_b32 s10, s7, 31
+; GCN-NEXT: s_mov_b32 s11, 0
+; GCN-NEXT: s_or_b64 s[8:9], s[2:3], s[10:11]
+; GCN-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
+; GCN-NEXT: s_lshr_b32 s10, s5, 31
+; GCN-NEXT: s_or_b64 s[6:7], s[0:1], s[10:11]
+; GCN-NEXT: s_mov_b32 s0, s6
+; GCN-NEXT: s_mov_b32 s2, s8
+; GCN-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i128_65:
; GFX11: ; %bb.0:
@@ -6547,6 +6706,24 @@ define amdgpu_ps i128 @s_fshl_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_mov_b32 s0, s6
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshl_i128_65:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
+; GFX9-NEXT: s_lshr_b32 s4, s5, 31
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
+; GFX9-NEXT: s_or_b32 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s4, s7, 31
+; GFX9-NEXT: s_or_b32 s2, s2, s4
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshl_i128_65:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
+; GFX10-NEXT: s_lshr_b32 s4, s5, 31
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
+; GFX10-NEXT: s_lshr_b32 s5, s7, 31
+; GFX10-NEXT: s_or_b32 s0, s0, s4
+; GFX10-NEXT: s_or_b32 s2, s2, s5
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -6574,17 +6751,27 @@ define i128 @v_fshl_i128_65(i128 %lhs, i128 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_i128_65:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[6:7]
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 31, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i128_65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 31, v7
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[6:7]
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 31, v5
; GFX9-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 31, v7
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_i128_65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6595,18 +6782,6 @@ define i128 @v_fshl_i128_65(i128 %lhs, i128 %rhs) {
; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
; GFX10-NEXT: v_or_b32_e32 v2, v2, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_i128_65:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[6:7]
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 31, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 31, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -6696,84 +6871,6 @@ define amdgpu_ps <2 x i128> @s_fshl_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshl_v2i128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s18, s20, 64
-; GFX9-NEXT: s_mov_b32 s19, 0
-; GFX9-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX9-NEXT: s_cselect_b32 s23, s15, s5
-; GFX9-NEXT: s_cselect_b32 s22, s14, s4
-; GFX9-NEXT: s_cselect_b32 s5, s5, s7
-; GFX9-NEXT: s_cselect_b32 s4, s4, s6
-; GFX9-NEXT: s_cselect_b32 s13, s13, s15
-; GFX9-NEXT: s_cselect_b32 s12, s12, s14
-; GFX9-NEXT: s_lshr_b64 s[6:7], s[22:23], 1
-; GFX9-NEXT: s_not_b32 s14, s20
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], s20
-; GFX9-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
-; GFX9-NEXT: s_lshr_b64 s[12:13], s[12:13], 1
-; GFX9-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[22:23], s20
-; GFX9-NEXT: s_lshr_b64 s[12:13], s[12:13], s14
-; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
-; GFX9-NEXT: s_and_b32 s18, s16, 64
-; GFX9-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX9-NEXT: s_cselect_b32 s13, s11, s1
-; GFX9-NEXT: s_cselect_b32 s12, s10, s0
-; GFX9-NEXT: s_cselect_b32 s1, s1, s3
-; GFX9-NEXT: s_cselect_b32 s0, s0, s2
-; GFX9-NEXT: s_cselect_b32 s9, s9, s11
-; GFX9-NEXT: s_cselect_b32 s8, s8, s10
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[12:13], 1
-; GFX9-NEXT: s_not_b32 s10, s16
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s16
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
-; GFX9-NEXT: s_lshr_b64 s[8:9], s[8:9], 1
-; GFX9-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[12:13], s16
-; GFX9-NEXT: s_lshr_b64 s[8:9], s[8:9], s10
-; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshl_v2i128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s18, s20, 64
-; GFX10-NEXT: s_mov_b32 s19, 0
-; GFX10-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX10-NEXT: s_cselect_b32 s23, s15, s5
-; GFX10-NEXT: s_cselect_b32 s22, s14, s4
-; GFX10-NEXT: s_cselect_b32 s5, s5, s7
-; GFX10-NEXT: s_cselect_b32 s4, s4, s6
-; GFX10-NEXT: s_cselect_b32 s13, s13, s15
-; GFX10-NEXT: s_cselect_b32 s12, s12, s14
-; GFX10-NEXT: s_lshr_b64 s[6:7], s[22:23], 1
-; GFX10-NEXT: s_not_b32 s14, s20
-; GFX10-NEXT: s_lshl_b64 s[4:5], s[4:5], s20
-; GFX10-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
-; GFX10-NEXT: s_and_b32 s18, s16, 64
-; GFX10-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
-; GFX10-NEXT: s_lshr_b64 s[4:5], s[12:13], 1
-; GFX10-NEXT: s_lshl_b64 s[12:13], s[22:23], s20
-; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s14
-; GFX10-NEXT: s_or_b64 s[4:5], s[12:13], s[4:5]
-; GFX10-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX10-NEXT: s_cselect_b32 s13, s11, s1
-; GFX10-NEXT: s_cselect_b32 s12, s10, s0
-; GFX10-NEXT: s_cselect_b32 s1, s1, s3
-; GFX10-NEXT: s_cselect_b32 s0, s0, s2
-; GFX10-NEXT: s_cselect_b32 s3, s9, s11
-; GFX10-NEXT: s_cselect_b32 s2, s8, s10
-; GFX10-NEXT: s_lshl_b64 s[8:9], s[0:1], s16
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[12:13], 1
-; GFX10-NEXT: s_not_b32 s14, s16
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX10-NEXT: s_lshl_b64 s[10:11], s[12:13], s16
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s14
-; GFX10-NEXT: s_lshr_b64 s[12:13], s[0:1], s14
-; GFX10-NEXT: s_or_b64 s[0:1], s[10:11], s[2:3]
-; GFX10-NEXT: s_or_b64 s[2:3], s[8:9], s[12:13]
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: s_fshl_v2i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s18, s20, 64
@@ -6814,6 +6911,182 @@ define amdgpu_ps <2 x i128> @s_fshl_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX11-NEXT: s_or_b64 s[0:1], s[10:11], s[2:3]
; GFX11-NEXT: s_or_b64 s[2:3], s[8:9], s[12:13]
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshl_v2i128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX9-NEXT: s_sub_i32 s19, s17, 64
+; GFX9-NEXT: s_sub_i32 s21, 64, s17
+; GFX9-NEXT: s_cmp_lt_u32 s17, 64
+; GFX9-NEXT: s_cselect_b32 s28, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s17, 0
+; GFX9-NEXT: s_cselect_b32 s17, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[24:25], s[0:1], s21
+; GFX9-NEXT: s_lshl_b64 s[26:27], s[2:3], s16
+; GFX9-NEXT: s_lshl_b64 s[22:23], s[0:1], s16
+; GFX9-NEXT: s_or_b64 s[24:25], s[24:25], s[26:27]
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s19
+; GFX9-NEXT: s_cmp_lg_u32 s28, 0
+; GFX9-NEXT: s_cselect_b64 s[22:23], s[22:23], 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[24:25], s[0:1]
+; GFX9-NEXT: s_cmp_lg_u32 s17, 0
+; GFX9-NEXT: s_mov_b32 s18, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
+; GFX9-NEXT: s_lshl_b32 s19, s10, 31
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[10:11], 1
+; GFX9-NEXT: s_andn2_b32 s10, 0x7f, s16
+; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[18:19]
+; GFX9-NEXT: s_not_b32 s17, s16
+; GFX9-NEXT: s_sub_i32 s19, s10, 64
+; GFX9-NEXT: s_sub_i32 s21, 64, s10
+; GFX9-NEXT: s_cmp_lt_u32 s10, 64
+; GFX9-NEXT: s_cselect_b32 s26, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s10, 0
+; GFX9-NEXT: s_cselect_b32 s27, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[8:9], s17
+; GFX9-NEXT: s_lshr_b64 s[16:17], s[0:1], s17
+; GFX9-NEXT: s_lshl_b64 s[24:25], s[8:9], s21
+; GFX9-NEXT: s_or_b64 s[16:17], s[16:17], s[24:25]
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[8:9], s19
+; GFX9-NEXT: s_cmp_lg_u32 s26, 0
+; GFX9-NEXT: s_cselect_b64 s[8:9], s[16:17], s[8:9]
+; GFX9-NEXT: s_cmp_lg_u32 s27, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX9-NEXT: s_cmp_lg_u32 s26, 0
+; GFX9-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX9-NEXT: s_and_b32 s8, s20, 0x7f
+; GFX9-NEXT: s_or_b64 s[0:1], s[22:23], s[0:1]
+; GFX9-NEXT: s_sub_i32 s19, s8, 64
+; GFX9-NEXT: s_sub_i32 s10, 64, s8
+; GFX9-NEXT: s_cmp_lt_u32 s8, 64
+; GFX9-NEXT: s_cselect_b32 s21, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s8, 0
+; GFX9-NEXT: s_cselect_b32 s22, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[4:5], s10
+; GFX9-NEXT: s_lshl_b64 s[16:17], s[6:7], s20
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[4:5], s20
+; GFX9-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], s19
+; GFX9-NEXT: s_cmp_lg_u32 s21, 0
+; GFX9-NEXT: s_cselect_b64 s[8:9], s[8:9], 0
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[10:11], s[4:5]
+; GFX9-NEXT: s_cmp_lg_u32 s22, 0
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[12:13], 1
+; GFX9-NEXT: s_lshl_b32 s19, s14, 31
+; GFX9-NEXT: s_andn2_b32 s12, 0x7f, s20
+; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], s[18:19]
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[14:15], 1
+; GFX9-NEXT: s_not_b32 s14, s20
+; GFX9-NEXT: s_sub_i32 s18, s12, 64
+; GFX9-NEXT: s_sub_i32 s16, 64, s12
+; GFX9-NEXT: s_cmp_lt_u32 s12, 64
+; GFX9-NEXT: s_cselect_b32 s19, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s12, 0
+; GFX9-NEXT: s_cselect_b32 s20, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[12:13], s[10:11], s14
+; GFX9-NEXT: s_lshr_b64 s[14:15], s[4:5], s14
+; GFX9-NEXT: s_lshl_b64 s[16:17], s[10:11], s16
+; GFX9-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[10:11], s18
+; GFX9-NEXT: s_cmp_lg_u32 s19, 0
+; GFX9-NEXT: s_cselect_b64 s[10:11], s[14:15], s[10:11]
+; GFX9-NEXT: s_cmp_lg_u32 s20, 0
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[4:5], s[10:11]
+; GFX9-NEXT: s_cmp_lg_u32 s19, 0
+; GFX9-NEXT: s_cselect_b64 s[10:11], s[12:13], 0
+; GFX9-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5]
+; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshl_v2i128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX10-NEXT: s_mov_b32 s18, 0
+; GFX10-NEXT: s_sub_i32 s19, s17, 64
+; GFX10-NEXT: s_sub_i32 s21, 64, s17
+; GFX10-NEXT: s_cmp_lt_u32 s17, 64
+; GFX10-NEXT: s_cselect_b32 s28, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s17, 0
+; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[22:23], s[0:1], s21
+; GFX10-NEXT: s_lshl_b64 s[24:25], s[2:3], s16
+; GFX10-NEXT: s_lshl_b64 s[26:27], s[0:1], s16
+; GFX10-NEXT: s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s19
+; GFX10-NEXT: s_cmp_lg_u32 s28, 0
+; GFX10-NEXT: s_cselect_b64 s[24:25], s[26:27], 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[22:23], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s17, 0
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
+; GFX10-NEXT: s_lshl_b32 s19, s10, 31
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[10:11], 1
+; GFX10-NEXT: s_andn2_b32 s10, 0x7f, s16
+; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[18:19]
+; GFX10-NEXT: s_not_b32 s19, s16
+; GFX10-NEXT: s_sub_i32 s21, s10, 64
+; GFX10-NEXT: s_sub_i32 s16, 64, s10
+; GFX10-NEXT: s_cmp_lt_u32 s10, 64
+; GFX10-NEXT: s_cselect_b32 s26, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s10, 0
+; GFX10-NEXT: s_cselect_b32 s27, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[10:11], s[0:1], s19
+; GFX10-NEXT: s_lshl_b64 s[16:17], s[8:9], s16
+; GFX10-NEXT: s_lshr_b64 s[22:23], s[8:9], s19
+; GFX10-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s21
+; GFX10-NEXT: s_cmp_lg_u32 s26, 0
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[10:11], s[8:9]
+; GFX10-NEXT: s_cmp_lg_u32 s27, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX10-NEXT: s_cmp_lg_u32 s26, 0
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[22:23], 0
+; GFX10-NEXT: s_and_b32 s10, s20, 0x7f
+; GFX10-NEXT: s_or_b64 s[0:1], s[24:25], s[0:1]
+; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX10-NEXT: s_sub_i32 s19, s10, 64
+; GFX10-NEXT: s_sub_i32 s8, 64, s10
+; GFX10-NEXT: s_cmp_lt_u32 s10, 64
+; GFX10-NEXT: s_cselect_b32 s21, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s10, 0
+; GFX10-NEXT: s_cselect_b32 s22, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[4:5], s8
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], s20
+; GFX10-NEXT: s_lshl_b64 s[16:17], s[4:5], s20
+; GFX10-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_lshl_b64 s[4:5], s[4:5], s19
+; GFX10-NEXT: s_cmp_lg_u32 s21, 0
+; GFX10-NEXT: s_cselect_b64 s[10:11], s[16:17], 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX10-NEXT: s_cmp_lg_u32 s22, 0
+; GFX10-NEXT: s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[12:13], 1
+; GFX10-NEXT: s_lshl_b32 s19, s14, 31
+; GFX10-NEXT: s_andn2_b32 s12, 0x7f, s20
+; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[18:19]
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[14:15], 1
+; GFX10-NEXT: s_not_b32 s16, s20
+; GFX10-NEXT: s_sub_i32 s18, s12, 64
+; GFX10-NEXT: s_sub_i32 s14, 64, s12
+; GFX10-NEXT: s_cmp_lt_u32 s12, 64
+; GFX10-NEXT: s_cselect_b32 s19, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s20, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[12:13], s[4:5], s16
+; GFX10-NEXT: s_lshl_b64 s[14:15], s[8:9], s14
+; GFX10-NEXT: s_lshr_b64 s[16:17], s[8:9], s16
+; GFX10-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s18
+; GFX10-NEXT: s_cmp_lg_u32 s19, 0
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX10-NEXT: s_cmp_lg_u32 s20, 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX10-NEXT: s_cmp_lg_u32 s19, 0
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[16:17], 0
+; GFX10-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT: ; return to shader part epilog
%result = call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -7001,6 +7274,96 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX8-NEXT: v_or_b32_e32 v7, v7, v11
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshl_v2i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v19, 0x7f, v16
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
+; GFX11-NEXT: v_bfi_b32 v29, v16, 0, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[17:18], v19, v[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v19
+; GFX11-NEXT: v_lshl_or_b32 v9, v10, 31, v9
+; GFX11-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v29
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v29
+; GFX11-NEXT: v_cndmask_b32_e32 v30, 0, v17, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v21, 64, v19
+; GFX11-NEXT: v_dual_cndmask_b32 v18, 0, v18 :: v_dual_add_nc_u32 v25, 0xffffffc0, v19
+; GFX11-NEXT: v_lshlrev_b64 v[23:24], v19, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[27:28], v16, v[10:11]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshrrev_b64 v[21:22], v21, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v25, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[25:26], v29, v[8:9]
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v19
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v21, v21, v23
+; GFX11-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v29
+; GFX11-NEXT: v_or_b32_e32 v19, v25, v27
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v0, v21, vcc_lo
+; GFX11-NEXT: v_or_b32_e32 v0, v22, v24
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v23, v[10:11]
+; GFX11-NEXT: v_or_b32_e32 v22, v26, v28
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v2, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v23, v1, v0, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29
+; GFX11-NEXT: v_cndmask_b32_e64 v16, v16, v19, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v29, v[10:11]
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v17, v22, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v22, v23, v3, s0
+; GFX11-NEXT: v_and_b32_e32 v23, 0x7f, v20
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v16, v8, vcc_lo
+; GFX11-NEXT: v_bfi_b32 v20, v20, 0, 0x7f
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v9, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
+; GFX11-NEXT: v_sub_nc_u32_e32 v10, 64, v23
+; GFX11-NEXT: v_cndmask_b32_e64 v24, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v25, 0, v1, s1
+; GFX11-NEXT: v_or_b32_e32 v0, v30, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v23
+; GFX11-NEXT: v_lshl_or_b32 v9, v14, 31, v9
+; GFX11-NEXT: v_lshrrev_b64 v[14:15], 1, v[14:15]
+; GFX11-NEXT: v_add_nc_u32_e32 v26, 0xffffffc0, v20
+; GFX11-NEXT: v_or_b32_e32 v1, v18, v3
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v10, v[4:5]
+; GFX11-NEXT: v_lshlrev_b64 v[10:11], v23, v[6:7]
+; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v20
+; GFX11-NEXT: v_lshlrev_b64 v[12:13], v23, v[4:5]
+; GFX11-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v23
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v20, v[8:9]
+; GFX11-NEXT: v_or_b32_e32 v10, v2, v10
+; GFX11-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
+; GFX11-NEXT: v_or_b32_e32 v2, v21, v24
+; GFX11-NEXT: v_or_b32_e32 v11, v3, v11
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_dual_cndmask_b32 v12, 0, v12 :: v_dual_cndmask_b32 v21, v4, v10
+; GFX11-NEXT: v_lshrrev_b64 v[3:4], v26, v[14:15]
+; GFX11-NEXT: v_or_b32_e32 v10, v16, v18
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v20
+; GFX11-NEXT: v_or_b32_e32 v16, v17, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v23
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v20
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v10, s1
+; GFX11-NEXT: v_lshrrev_b64 v[10:11], v20, v[14:15]
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v16, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v21, v6, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v5, v7, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v3, v8, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v4, v9, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v10, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v10, 0, v11, s1
+; GFX11-NEXT: v_or_b32_e32 v3, v22, v25
+; GFX11-NEXT: v_or_b32_e32 v4, v12, v5
+; GFX11-NEXT: v_or_b32_e32 v5, v13, v8
+; GFX11-NEXT: v_or_b32_e32 v6, v6, v9
+; GFX11-NEXT: v_or_b32_e32 v7, v7, v10
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7088,7 +7451,6 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX9-NEXT: v_or_b32_e32 v6, v17, v6
; GFX9-NEXT: v_or_b32_e32 v7, v12, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshl_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7175,97 +7537,6 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX10-NEXT: v_or_b32_e32 v6, v6, v9
; GFX10-NEXT: v_or_b32_e32 v7, v7, v10
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_v2i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v19, 0x7f, v16
-; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX11-NEXT: v_bfi_b32 v29, v16, 0, 0x7f
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[17:18], v19, v[0:1]
-; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v19
-; GFX11-NEXT: v_lshl_or_b32 v9, v10, 31, v9
-; GFX11-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v29
-; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v29
-; GFX11-NEXT: v_cndmask_b32_e32 v30, 0, v17, vcc_lo
-; GFX11-NEXT: v_sub_nc_u32_e32 v21, 64, v19
-; GFX11-NEXT: v_dual_cndmask_b32 v18, 0, v18 :: v_dual_add_nc_u32 v25, 0xffffffc0, v19
-; GFX11-NEXT: v_lshlrev_b64 v[23:24], v19, v[2:3]
-; GFX11-NEXT: v_lshlrev_b64 v[27:28], v16, v[10:11]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[21:22], v21, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v25, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[25:26], v29, v[8:9]
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v19
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v21, v21, v23
-; GFX11-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v29
-; GFX11-NEXT: v_or_b32_e32 v19, v25, v27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v21, v0, v21, vcc_lo
-; GFX11-NEXT: v_or_b32_e32 v0, v22, v24
-; GFX11-NEXT: v_lshrrev_b64 v[16:17], v23, v[10:11]
-; GFX11-NEXT: v_or_b32_e32 v22, v26, v28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v2, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v23, v1, v0, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29
-; GFX11-NEXT: v_cndmask_b32_e64 v16, v16, v19, s1
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v29, v[10:11]
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v17, v22, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v22, v23, v3, s0
-; GFX11-NEXT: v_and_b32_e32 v23, 0x7f, v20
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v16, v8, vcc_lo
-; GFX11-NEXT: v_bfi_b32 v20, v20, 0, 0x7f
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v9, vcc_lo
-; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
-; GFX11-NEXT: v_sub_nc_u32_e32 v10, 64, v23
-; GFX11-NEXT: v_cndmask_b32_e64 v24, 0, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v25, 0, v1, s1
-; GFX11-NEXT: v_or_b32_e32 v0, v30, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v23
-; GFX11-NEXT: v_lshl_or_b32 v9, v14, 31, v9
-; GFX11-NEXT: v_lshrrev_b64 v[14:15], 1, v[14:15]
-; GFX11-NEXT: v_add_nc_u32_e32 v26, 0xffffffc0, v20
-; GFX11-NEXT: v_or_b32_e32 v1, v18, v3
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v10, v[4:5]
-; GFX11-NEXT: v_lshlrev_b64 v[10:11], v23, v[6:7]
-; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v20
-; GFX11-NEXT: v_lshlrev_b64 v[12:13], v23, v[4:5]
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
-; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v23
-; GFX11-NEXT: v_lshrrev_b64 v[16:17], v20, v[8:9]
-; GFX11-NEXT: v_or_b32_e32 v10, v2, v10
-; GFX11-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
-; GFX11-NEXT: v_or_b32_e32 v2, v21, v24
-; GFX11-NEXT: v_or_b32_e32 v11, v3, v11
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_dual_cndmask_b32 v12, 0, v12 :: v_dual_cndmask_b32 v21, v4, v10
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], v26, v[14:15]
-; GFX11-NEXT: v_or_b32_e32 v10, v16, v18
-; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v20
-; GFX11-NEXT: v_or_b32_e32 v16, v17, v19
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v23
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v20
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v10, s1
-; GFX11-NEXT: v_lshrrev_b64 v[10:11], v20, v[14:15]
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v16, s1
-; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v21, v6, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v5, v7, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v3, v8, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v4, v9, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v10, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v10, 0, v11, s1
-; GFX11-NEXT: v_or_b32_e32 v3, v22, v25
-; GFX11-NEXT: v_or_b32_e32 v4, v12, v5
-; GFX11-NEXT: v_or_b32_e32 v5, v13, v8
-; GFX11-NEXT: v_or_b32_e32 v6, v6, v9
-; GFX11-NEXT: v_or_b32_e32 v7, v7, v10
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -7302,3 +7573,5 @@ declare i128 @llvm.fshl.i128(i128, i128, i128) #0
declare <2 x i128> @llvm.fshl.v2i128(<2 x i128>, <2 x i128>, <2 x i128>) #0
attributes #0 = { nounwind readnone speculatable willreturn }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 2d6d662777526..bccd8958962da 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX6-LABEL: s_fshr_i7:
@@ -80,6 +80,47 @@ define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i7:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, -7
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_i32 s3, s3, -7
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
+; GFX11-NEXT: s_sub_i32 s3, 6, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -114,7 +155,6 @@ define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -149,48 +189,6 @@ define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i7:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, -7
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_i32 s3, s3, -7
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX11-NEXT: s_sub_i32 s3, 6, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i7 @llvm.fshr.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
}
@@ -243,70 +241,117 @@ define i7 @v_fshr_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX8-NEXT: v_lshrrev_b16_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i7:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v2.l, 0x7f, v2.l
+; GFX11-NEXT: v_and_b16 v0.h, 0x7f, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 7, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_u32_u16_e32 v3, v2.l
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_u32_u24_e32 v3, 0x2493, v3
+; GFX11-NEXT: v_mul_lo_u16 v1.l, v3.h, 7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_nc_u16 v0.h, v2.l, v1.l
+; GFX11-NEXT: v_lshrrev_b16 v0.l, v0.h, v0.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 1, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0x7f, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 7, v0
-; GFX9-NEXT: s_movk_i32 s4, 0x2493
+; GFX9-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_readfirstlane_b32 s4, v3
+; GFX9-NEXT: s_mul_i32 s5, s4, -7
+; GFX9-NEXT: s_mul_hi_u32 s5, s4, s5
+; GFX9-NEXT: s_add_i32 s4, s4, s5
+; GFX9-NEXT: v_mul_hi_u32 v3, v2, s4
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, -7, v[2:3]
+; GFX9-NEXT: v_add_u32_e32 v3, -7, v2
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_add_u32_e32 v3, -7, v2
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_sub_u16_e32 v3, 6, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX9-NEXT: v_and_b32_e32 v3, 0x7f, v3
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, v3, v0
+; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_u32_u24_sdwa v1, v2, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v3, 7
-; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_sub_u16_e32 v1, v2, v1
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
; GFX10-NEXT: v_and_b32_e32 v2, 0x7f, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x2493
+; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0x7f, v1
-; GFX10-NEXT: v_lshlrev_b16 v0, 7, v0
-; GFX10-NEXT: v_mul_u32_u24_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT: v_readfirstlane_b32 s4, v3
+; GFX10-NEXT: s_mul_i32 s5, s4, -7
+; GFX10-NEXT: s_mul_hi_u32 s5, s4, s5
+; GFX10-NEXT: s_add_i32 s4, s4, s5
+; GFX10-NEXT: v_mul_hi_u32 v3, v2, s4
+; GFX10-NEXT: v_mad_u64_u32 v[2:3], s4, v3, -7, v[2:3]
+; GFX10-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT: v_sub_nc_u16 v3, 6, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX10-NEXT: v_and_b32_e32 v3, 0x7f, v3
+; GFX10-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX10-NEXT: v_lshlrev_b16 v0, v3, v0
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX10-NEXT: v_mul_lo_u16 v3, v3, 7
-; GFX10-NEXT: v_sub_nc_u16 v1, v2, v3
-; GFX10-NEXT: v_lshrrev_b16 v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i7:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0x7f, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0x7f, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 7, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mul_u32_u24_e32 v3, 0x2493, v3
-; GFX11-TRUE16-NEXT: v_mul_lo_u16 v1.l, v3.h, 7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.h, v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_i7:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 1, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0x7f, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 7, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_mul_u32_u24_e32 v3, 0x2493, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-FAKE16-NEXT: s_mul_i32 s1, s0, -7
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, s1
+; GFX11-FAKE16-NEXT: v_mul_hi_u32 v5, v2, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_mul_lo_u16 v3, v3, 7
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v1, v2, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, -7, v3
+; GFX11-FAKE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX11-FAKE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_nc_u16 v3, 6, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0x7f, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v3, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i7 @llvm.fshr.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
@@ -337,45 +382,41 @@ define amdgpu_ps i8 @s_fshr_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_xor_b32 s3, s2, -1
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s3, s3, 7
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s3, s2, -1
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: s_and_b32 s3, s3, 7
+; GFX9-NEXT: s_andn2_b32 s3, 7, s2
; GFX9-NEXT: s_and_b32 s2, s2, 7
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
; GFX9-NEXT: s_lshl_b32 s0, s0, s3
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s3, s2, -1
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
; GFX10-NEXT: s_lshl_b32 s0, s0, 1
-; GFX10-NEXT: s_and_b32 s3, s3, 7
+; GFX10-NEXT: s_andn2_b32 s3, 7, s2
; GFX10-NEXT: s_and_b32 s2, s2, 7
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
; GFX10-NEXT: s_lshl_b32 s0, s0, s3
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_xor_b32 s3, s2, -1
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_and_b32 s3, s3, 7
-; GFX11-NEXT: s_and_b32 s2, s2, 7
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
}
@@ -402,44 +443,52 @@ define i8 @v_fshr_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX8-NEXT: v_lshrrev_b16_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: v_and_b16 v0.h, v2.l, 7
+; GFX11-NEXT: v_lshrrev_b16 v0.l, v0.h, v0.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, v1, v0
+; GFX9-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, v3, v0
+; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX10-NEXT: v_lshrrev_b16 v0, v1, v0
+; GFX10-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
+; GFX10-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX10-NEXT: v_lshlrev_b16 v0, v3, v0
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v2.l, 7
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 1, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 7, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v3, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
@@ -462,6 +511,15 @@ define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i8_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -470,7 +528,6 @@ define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -479,16 +536,6 @@ define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 4
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i8_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 4
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 4)
ret i8 %result
}
@@ -511,6 +558,15 @@ define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i8_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -519,7 +575,6 @@ define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -528,17 +583,6 @@ define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 4, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i8_4:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_i8_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -569,6 +613,15 @@ define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i8_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 5
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -577,7 +630,6 @@ define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 5
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -586,16 +638,6 @@ define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 5
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i8_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 3
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 5
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 5)
ret i8 %result
}
@@ -618,6 +660,15 @@ define i8 @v_fshr_i8_5(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i8_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 3, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 5, v0.h
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -626,7 +677,6 @@ define i8 @v_fshr_i8_5(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -635,17 +685,6 @@ define i8 @v_fshr_i8_5(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 5, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i8_5:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 3, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 5, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_i8_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -713,64 +752,6 @@ define amdgpu_ps i16 @s_fshr_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshr_v2i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s4, s1, 8
-; GFX9-NEXT: s_xor_b32 s6, s2, -1
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_lshr_b32 s3, s0, 8
-; GFX9-NEXT: s_lshr_b32 s5, s2, 8
-; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: s_and_b32 s6, s6, 7
-; GFX9-NEXT: s_and_b32 s2, s2, 7
-; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX9-NEXT: s_lshl_b32 s0, s0, s6
-; GFX9-NEXT: s_lshr_b32 s1, s1, s2
-; GFX9-NEXT: s_xor_b32 s2, s5, -1
-; GFX9-NEXT: s_or_b32 s0, s0, s1
-; GFX9-NEXT: s_lshl_b32 s1, s3, 1
-; GFX9-NEXT: s_and_b32 s2, s2, 7
-; GFX9-NEXT: s_and_b32 s3, s4, 0xff
-; GFX9-NEXT: s_lshl_b32 s1, s1, s2
-; GFX9-NEXT: s_and_b32 s2, s5, 7
-; GFX9-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX9-NEXT: s_lshr_b32 s2, s3, s2
-; GFX9-NEXT: s_or_b32 s1, s1, s2
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_lshl_b32 s1, s1, 8
-; GFX9-NEXT: s_or_b32 s0, s0, s1
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshr_v2i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_xor_b32 s5, s2, -1
-; GFX10-NEXT: s_lshr_b32 s3, s0, 8
-; GFX10-NEXT: s_lshr_b32 s4, s1, 8
-; GFX10-NEXT: s_lshl_b32 s0, s0, 1
-; GFX10-NEXT: s_and_b32 s5, s5, 7
-; GFX10-NEXT: s_lshr_b32 s6, s2, 8
-; GFX10-NEXT: s_lshl_b32 s0, s0, s5
-; GFX10-NEXT: s_xor_b32 s5, s6, -1
-; GFX10-NEXT: s_and_b32 s4, s4, 0xff
-; GFX10-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10-NEXT: s_lshl_b32 s3, s3, 1
-; GFX10-NEXT: s_and_b32 s5, s5, 7
-; GFX10-NEXT: s_and_b32 s6, s6, 7
-; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
-; GFX10-NEXT: s_and_b32 s2, s2, 7
-; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX10-NEXT: s_lshl_b32 s3, s3, s5
-; GFX10-NEXT: s_lshr_b32 s4, s4, s6
-; GFX10-NEXT: s_lshr_b32 s1, s1, s2
-; GFX10-NEXT: s_or_b32 s2, s3, s4
-; GFX10-NEXT: s_or_b32 s0, s0, s1
-; GFX10-NEXT: s_and_b32 s1, s2, 0xff
-; GFX10-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10-NEXT: s_lshl_b32 s1, s1, 8
-; GFX10-NEXT: s_or_b32 s0, s0, s1
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: s_fshr_v2i8:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_xor_b32 s5, s2, -1
@@ -800,6 +781,58 @@ define amdgpu_ps i16 @s_fshr_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshr_v2i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s4, s1, 8
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_lshr_b32 s3, s0, 8
+; GFX9-NEXT: s_lshr_b32 s5, s2, 8
+; GFX9-NEXT: s_lshl_b32 s0, s0, 1
+; GFX9-NEXT: s_andn2_b32 s6, 7, s2
+; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s6
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s1, s3, 1
+; GFX9-NEXT: s_andn2_b32 s2, 7, s5
+; GFX9-NEXT: s_and_b32 s3, s4, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s2, s5, 7
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX9-NEXT: s_lshr_b32 s2, s3, s2
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshr_v2i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshr_b32 s4, s1, 8
+; GFX10-NEXT: s_lshr_b32 s3, s0, 8
+; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_andn2_b32 s5, 7, s2
+; GFX10-NEXT: s_lshr_b32 s6, s2, 8
+; GFX10-NEXT: s_and_b32 s4, s4, 0xff
+; GFX10-NEXT: s_lshl_b32 s0, s0, s5
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s3, s3, 1
+; GFX10-NEXT: s_andn2_b32 s5, 7, s6
+; GFX10-NEXT: s_and_b32 s6, s6, 7
+; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX10-NEXT: s_and_b32 s2, s2, 7
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshl_b32 s3, s3, s5
+; GFX10-NEXT: s_lshr_b32 s4, s4, s6
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
+; GFX10-NEXT: s_or_b32 s2, s3, s4
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: s_and_b32 s1, s2, 0xff
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%amt = bitcast i16 %amt.arg to <2 x i8>
@@ -848,75 +881,107 @@ define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v2i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
+; GFX11-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
+; GFX11-NEXT: v_and_b16 v2.h, 0xff00, v0.l
+; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-NEXT: v_and_b16 v0.h, v0.h, 7
+; GFX11-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-NEXT: v_lshrrev_b16 v0.h, v0.h, v1.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b16 v0.l, v2.l, v0.l
+; GFX11-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 8, 3
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v1, 7, v2
-; GFX9-NEXT: v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, v1, v0
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_xor_b32_e32 v6, -1, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT: v_and_b32_e32 v6, 7, v6
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, v6, v0
+; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_xor_b32_e32 v2, -1, v5
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 1, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, v2, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 7, v5
+; GFX9-NEXT: v_lshrrev_b16_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX10-NEXT: v_bfe_u32 v4, v2, 8, 3
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-NEXT: v_xor_b32_e32 v7, -1, v2
+; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
+; GFX10-NEXT: v_xor_b32_e32 v6, -1, v3
+; GFX10-NEXT: v_lshlrev_b16 v4, 1, v4
+; GFX10-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xff, v5
; GFX10-NEXT: v_and_b32_e32 v2, 7, v2
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_lshrrev_b16 v3, v4, v3
-; GFX10-NEXT: v_lshrrev_b16 v0, v2, v0
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 7, v6
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX10-NEXT: v_lshrrev_b16 v3, v3, v5
+; GFX10-NEXT: v_lshlrev_b16 v4, v6, v4
+; GFX10-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX10-NEXT: v_lshlrev_b16 v0, v7, v0
+; GFX10-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_and_b32_sdwa v1, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_v2i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff00, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffffff00, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 8, 3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v7, -1, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 1, v0
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v6, -1, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 1, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 7, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 7, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, v6, v4
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v7, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v5, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v2, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i16 %lhs.arg to <2 x i8>
@@ -993,59 +1058,6 @@ define amdgpu_ps i32 @s_fshr_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshr_v4i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, s0, v1
-; GFX9-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX9-NEXT: s_and_b32 s0, s2, 7
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, s0, v2
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0307
-; GFX9-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30018
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0206
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, s0, v2
-; GFX9-NEXT: v_perm_b32 v0, s1, v0, v3
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30010
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshr_v4i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xc0c0307
-; GFX10-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xc0c0206
-; GFX10-NEXT: s_bfe_u32 s3, s2, 0x30008
-; GFX10-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX10-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX10-NEXT: v_perm_b32 v2, s1, s0, v2
-; GFX10-NEXT: v_perm_b32 v3, s1, s0, v3
-; GFX10-NEXT: s_and_b32 s4, s2, 7
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s3, v0
-; GFX10-NEXT: s_bfe_u32 s3, s2, 0x30018
-; GFX10-NEXT: s_bfe_u32 s0, s2, 0x30010
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s3, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, s4, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, s0, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: s_fshr_v4i8:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0105
@@ -1083,6 +1095,110 @@ define amdgpu_ps i32 @s_fshr_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshr_v4i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s6, s1, 8
+; GFX9-NEXT: s_lshr_b32 s7, s1, 16
+; GFX9-NEXT: s_lshr_b32 s8, s1, 24
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_lshr_b32 s3, s0, 8
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshr_b32 s5, s0, 24
+; GFX9-NEXT: s_lshr_b32 s9, s2, 8
+; GFX9-NEXT: s_lshr_b32 s10, s2, 16
+; GFX9-NEXT: s_lshr_b32 s11, s2, 24
+; GFX9-NEXT: s_lshl_b32 s0, s0, 1
+; GFX9-NEXT: s_andn2_b32 s12, 7, s2
+; GFX9-NEXT: s_and_b32 s2, s2, 7
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_lshl_b32 s0, s0, s12
+; GFX9-NEXT: s_lshr_b32 s1, s1, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: s_lshl_b32 s1, s3, 1
+; GFX9-NEXT: s_andn2_b32 s2, 7, s9
+; GFX9-NEXT: s_and_b32 s3, s6, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s2, s9, 7
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX9-NEXT: s_lshr_b32 s2, s3, s2
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_lshl_b32 s2, s4, 1
+; GFX9-NEXT: s_andn2_b32 s3, 7, s10
+; GFX9-NEXT: s_and_b32 s4, s7, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, s3
+; GFX9-NEXT: s_and_b32 s3, s10, 7
+; GFX9-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX9-NEXT: s_lshr_b32 s3, s4, s3
+; GFX9-NEXT: s_or_b32 s2, s2, s3
+; GFX9-NEXT: s_lshl_b32 s3, s5, 1
+; GFX9-NEXT: s_andn2_b32 s4, 7, s11
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s3, s3, s4
+; GFX9-NEXT: s_and_b32 s4, s11, 7
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_lshr_b32 s4, s8, s4
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: s_and_b32 s1, s2, 0xff
+; GFX9-NEXT: s_or_b32 s3, s3, s4
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: s_and_b32 s1, s3, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshr_v4i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshr_b32 s6, s1, 8
+; GFX10-NEXT: s_lshr_b32 s7, s1, 16
+; GFX10-NEXT: s_lshr_b32 s8, s1, 24
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshr_b32 s3, s0, 8
+; GFX10-NEXT: s_lshr_b32 s9, s2, 8
+; GFX10-NEXT: s_lshr_b32 s10, s2, 16
+; GFX10-NEXT: s_lshr_b32 s11, s2, 24
+; GFX10-NEXT: s_andn2_b32 s12, 7, s2
+; GFX10-NEXT: s_and_b32 s2, s2, 7
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_and_b32 s6, s6, 0xff
+; GFX10-NEXT: s_lshr_b32 s4, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s0, 24
+; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_lshr_b32 s1, s1, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, 1
+; GFX10-NEXT: s_andn2_b32 s3, 7, s9
+; GFX10-NEXT: s_and_b32 s9, s9, 7
+; GFX10-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX10-NEXT: s_lshl_b32 s0, s0, s12
+; GFX10-NEXT: s_lshl_b32 s2, s2, s3
+; GFX10-NEXT: s_lshr_b32 s3, s6, s9
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: s_or_b32 s1, s2, s3
+; GFX10-NEXT: s_lshl_b32 s2, s4, 1
+; GFX10-NEXT: s_and_b32 s4, s7, 0xff
+; GFX10-NEXT: s_andn2_b32 s3, 7, s10
+; GFX10-NEXT: s_and_b32 s6, s10, 7
+; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX10-NEXT: s_lshl_b32 s2, s2, s3
+; GFX10-NEXT: s_lshr_b32 s3, s4, s6
+; GFX10-NEXT: s_lshl_b32 s4, s5, 1
+; GFX10-NEXT: s_andn2_b32 s5, 7, s11
+; GFX10-NEXT: s_and_b32 s6, s11, 7
+; GFX10-NEXT: s_lshl_b32 s4, s4, s5
+; GFX10-NEXT: s_lshr_b32 s5, s8, s6
+; GFX10-NEXT: s_or_b32 s2, s2, s3
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_or_b32 s3, s4, s5
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10-NEXT: s_and_b32 s2, s2, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: s_lshl_b32 s1, s2, 16
+; GFX10-NEXT: s_and_b32 s2, s3, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: s_lshl_b32 s1, s2, 24
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
%amt = bitcast i32 %amt.arg to <4 x i8>
@@ -1155,143 +1271,197 @@ define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX8-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v4i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b16 v3.l, 8, v1.l
+; GFX11-NEXT: v_and_b16 v3.h, 0xff00, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 24, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-NEXT: v_or_b16 v3.l, v3.h, v3.l
+; GFX11-NEXT: v_and_b16 v3.h, v4.l, 7
+; GFX11-NEXT: v_and_b16 v4.l, 0xff00, v0.h
+; GFX11-NEXT: v_and_b16 v1.h, 0xff, v1.h
+; GFX11-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-NEXT: v_and_b16 v1.l, v2.l, 7
+; GFX11-NEXT: v_or_b16 v2.l, v4.l, v6.l
+; GFX11-NEXT: v_and_b16 v4.l, v5.l, 7
+; GFX11-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-NEXT: v_and_b16 v1.h, v2.h, 7
+; GFX11-NEXT: v_lshrrev_b16 v2.h, v3.h, v3.l
+; GFX11-NEXT: v_lshrrev_b16 v0.l, v1.l, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v1.l, v4.l, v2.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.h, v0.h
+; GFX11-NEXT: v_lshlrev_b16 v1.h, 8, v2.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v1.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b16 v0.h, v0.h, v1.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 8, 3
-; GFX9-NEXT: v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v0
-; GFX9-NEXT: v_or_b32_sdwa v4, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v5, 7, v2
-; GFX9-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-NEXT: v_lshrrev_b16_e32 v4, v5, v4
-; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_sdwa v4, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v5, 7
-; GFX9-NEXT: v_or_b32_sdwa v4, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT: v_and_b32_sdwa v6, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_movk_i32 s5, 0xff
-; GFX9-NEXT: v_lshrrev_b16_sdwa v4, v6, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v6, 8
-; GFX9-NEXT: v_and_b32_sdwa v1, v1, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_xor_b32_e32 v7, -1, v2
+; GFX9-NEXT: v_lshlrev_b16_e32 v6, 1, v0
+; GFX9-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX9-NEXT: v_lshlrev_b16_e32 v6, v7, v6
+; GFX9-NEXT: v_and_b32_e32 v7, 7, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT: v_lshrrev_b16_sdwa v7, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT: v_or_b32_e32 v6, v6, v7
+; GFX9-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_lshlrev_b16_e32 v3, 1, v3
+; GFX9-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX9-NEXT: v_and_b32_e32 v5, 7, v5
+; GFX9-NEXT: v_lshlrev_b16_e32 v3, v7, v3
+; GFX9-NEXT: v_lshrrev_b16_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_mov_b32_e32 v7, -1
+; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX9-NEXT: v_mov_b32_e32 v4, 1
+; GFX9-NEXT: v_xor_b32_sdwa v9, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b16_sdwa v5, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_and_b32_e32 v9, 7, v9
+; GFX9-NEXT: v_mov_b32_e32 v8, 0xff
+; GFX9-NEXT: v_lshlrev_b16_e32 v5, v9, v5
+; GFX9-NEXT: v_mov_b32_e32 v9, 7
+; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_xor_b32_sdwa v4, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v10, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v11, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_e32 v4, 7, v4
+; GFX9-NEXT: v_and_b32_sdwa v2, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b16_e32 v10, v10, v11
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, v4, v0
+; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v10
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_sdwa v1, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, v1, v0
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v1, 8
+; GFX9-NEXT: v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-NEXT: v_and_or_b32 v1, v6, v8, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffffff00, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX10-NEXT: v_xor_b32_e32 v8, -1, v2
+; GFX10-NEXT: v_mov_b32_e32 v3, -1
; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, 7
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xff
-; GFX10-NEXT: v_bfe_u32 v7, v2, 8, 3
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_or_b32_sdwa v5, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffffff00, v6
-; GFX10-NEXT: v_and_b32_e32 v8, 7, v2
-; GFX10-NEXT: v_and_b32_sdwa v9, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-NEXT: v_lshrrev_b16 v5, v7, v5
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_or_b32_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX10-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_e32 v3, v6, v4
-; GFX10-NEXT: v_lshlrev_b16 v4, 8, v5
-; GFX10-NEXT: v_lshrrev_b16 v0, v8, v0
-; GFX10-NEXT: v_lshrrev_b16 v1, v9, v1
-; GFX10-NEXT: v_lshrrev_b16 v2, v2, v3
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_xor_b32_e32 v10, -1, v5
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
+; GFX10-NEXT: v_and_b32_e32 v8, 7, v8
+; GFX10-NEXT: v_lshlrev_b16 v4, 1, v4
+; GFX10-NEXT: v_and_b32_e32 v10, 7, v10
+; GFX10-NEXT: v_mov_b32_e32 v14, 0xff
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX10-NEXT: v_lshlrev_b16 v0, v8, v0
+; GFX10-NEXT: v_and_b32_e32 v8, 0xff, v9
+; GFX10-NEXT: v_lshlrev_b16 v4, v10, v4
+; GFX10-NEXT: v_xor_b32_sdwa v9, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_mov_b32_e32 v10, 7
+; GFX10-NEXT: v_xor_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v12, 7, v2
+; GFX10-NEXT: v_and_b32_e32 v13, 0xff, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 7, v5
+; GFX10-NEXT: v_lshlrev_b16 v6, 1, v6
+; GFX10-NEXT: v_and_b32_e32 v9, 7, v9
+; GFX10-NEXT: v_and_b32_sdwa v15, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_sdwa v1, v1, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_lshlrev_b16 v7, 1, v7
+; GFX10-NEXT: v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT: v_and_b32_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b16 v5, v5, v8
+; GFX10-NEXT: v_lshlrev_b16 v6, v9, v6
+; GFX10-NEXT: v_lshrrev_b16 v1, v15, v1
+; GFX10-NEXT: v_lshlrev_b16 v3, v3, v7
+; GFX10-NEXT: v_lshrrev_b16 v2, v2, v11
+; GFX10-NEXT: v_lshrrev_b16 v7, v12, v13
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX10-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v7
+; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX10-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_v4i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff00, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 7
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff00, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v2.l, 7
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v5.l, 7
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.h, 7
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v6, 8, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffffff00, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v9, 8, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 24, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 8, 3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v9, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 7, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffffff00, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 8, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 24, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v12, -1, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 1, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 7, v12
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v14, -1, v11
; GFX11-FAKE16-NEXT: v_lshrrev_b16 v6, v7, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, v2, v8
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v9, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 7, v5
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 7, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v5, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v3, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v7, -1, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 24, v1
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v10, -1, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, v12, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 1, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 7, v14
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 7, v11
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 1, v5
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 7, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 7, v13
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 1, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 7, v10
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 7, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, v12, v4
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v6, v11, v8
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, v7, v5
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v7, v13, v9
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v10, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v4, v6
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v5, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-FAKE16-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -1302,76 +1472,79 @@ define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
}
define amdgpu_ps i24 @s_fshr_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt) {
-; GFX6-LABEL: s_fshr_i24:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX6-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX6-NEXT: v_mul_hi_u32 v0, v0, s4
-; GFX6-NEXT: v_readfirstlane_b32 s4, v0
-; GFX6-NEXT: s_add_i32 s3, s3, s4
-; GFX6-NEXT: v_mov_b32_e32 v0, s3
-; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX6-NEXT: s_add_i32 s2, s2, s3
-; GFX6-NEXT: s_cmp_ge_u32 s2, 24
-; GFX6-NEXT: s_cselect_b32 s3, 1, 0
-; GFX6-NEXT: s_sub_i32 s4, s2, 24
-; GFX6-NEXT: s_cmp_lg_u32 s3, 0
-; GFX6-NEXT: s_cselect_b32 s2, s4, s2
-; GFX6-NEXT: s_cmp_ge_u32 s2, 24
-; GFX6-NEXT: s_cselect_b32 s3, 1, 0
-; GFX6-NEXT: s_sub_i32 s4, s2, 24
-; GFX6-NEXT: s_cmp_lg_u32 s3, 0
-; GFX6-NEXT: s_cselect_b32 s2, s4, s2
-; GFX6-NEXT: s_lshl_b32 s0, s0, 1
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffffff
-; GFX6-NEXT: s_sub_i32 s3, 23, s2
-; GFX6-NEXT: s_lshl_b32 s0, s0, s3
-; GFX6-NEXT: s_lshr_b32 s1, s1, s2
-; GFX6-NEXT: s_or_b32 s0, s0, s1
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshr_i24:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s3, v0
-; GFX8-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
-; GFX8-NEXT: v_readfirstlane_b32 s4, v0
-; GFX8-NEXT: s_add_i32 s3, s3, s4
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT: v_readfirstlane_b32 s3, v0
-; GFX8-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX8-NEXT: s_add_i32 s2, s2, s3
-; GFX8-NEXT: s_cmp_ge_u32 s2, 24
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-NEXT: s_sub_i32 s4, s2, 24
-; GFX8-NEXT: s_cmp_lg_u32 s3, 0
-; GFX8-NEXT: s_cselect_b32 s2, s4, s2
-; GFX8-NEXT: s_cmp_ge_u32 s2, 24
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-NEXT: s_sub_i32 s4, s2, 24
-; GFX8-NEXT: s_cmp_lg_u32 s3, 0
-; GFX8-NEXT: s_cselect_b32 s2, s4, s2
-; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffffff
-; GFX8-NEXT: s_sub_i32 s3, 23, s2
-; GFX8-NEXT: s_lshl_b32 s0, s0, s3
-; GFX8-NEXT: s_lshr_b32 s1, s1, s2
-; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshr_i24:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT: s_and_b32 s2, s2, 0xffffff
+; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT: v_readfirstlane_b32 s3, v0
+; GCN-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GCN-NEXT: v_mul_hi_u32 v0, v0, s4
+; GCN-NEXT: v_readfirstlane_b32 s4, v0
+; GCN-NEXT: s_add_i32 s3, s3, s4
+; GCN-NEXT: v_mov_b32_e32 v0, s3
+; GCN-NEXT: v_mul_hi_u32 v0, s2, v0
+; GCN-NEXT: v_readfirstlane_b32 s3, v0
+; GCN-NEXT: s_mulk_i32 s3, 0xffe8
+; GCN-NEXT: s_add_i32 s2, s2, s3
+; GCN-NEXT: s_cmp_ge_u32 s2, 24
+; GCN-NEXT: s_cselect_b32 s3, 1, 0
+; GCN-NEXT: s_sub_i32 s4, s2, 24
+; GCN-NEXT: s_cmp_lg_u32 s3, 0
+; GCN-NEXT: s_cselect_b32 s2, s4, s2
+; GCN-NEXT: s_cmp_ge_u32 s2, 24
+; GCN-NEXT: s_cselect_b32 s3, 1, 0
+; GCN-NEXT: s_sub_i32 s4, s2, 24
+; GCN-NEXT: s_cmp_lg_u32 s3, 0
+; GCN-NEXT: s_cselect_b32 s2, s4, s2
+; GCN-NEXT: s_lshl_b32 s0, s0, 1
+; GCN-NEXT: s_and_b32 s1, s1, 0xffffff
+; GCN-NEXT: s_sub_i32 s3, 23, s2
+; GCN-NEXT: s_lshl_b32 s0, s0, s3
+; GCN-NEXT: s_lshr_b32 s1, s1, s2
+; GCN-NEXT: s_or_b32 s0, s0, s1
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_sub_i32 s3, 23, s2
+; GFX11-NEXT: s_and_b32 s1, s1, 0xffffff
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1403,7 +1576,6 @@ define amdgpu_ps i24 @s_fshr_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1434,46 +1606,7 @@ define amdgpu_ps i24 @s_fshr_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX10-NEXT: s_lshl_b32 s0, s0, s3
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_sub_i32 s3, 23, s2
-; GFX11-NEXT: s_and_b32 s1, s1, 0xffffff
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
+; GFX10-NEXT: ; return to shader part epilog
%result = call i24 @llvm.fshr.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -1545,6 +1678,43 @@ define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v3, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1574,7 +1744,6 @@ define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, v2, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, v3, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1603,44 +1772,6 @@ define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX10-NEXT: v_lshrrev_b32_e32 v1, v2, v1
; GFX10-NEXT: v_lshl_or_b32 v0, v0, v3, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, v2, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v3, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i24 @llvm.fshr.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -1843,337 +1974,462 @@ define amdgpu_ps i48 @s_fshr_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX8-NEXT: v_readfirstlane_b32 s1, v1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_v2i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_mov_b32_e32 v2, s5
+; GFX11-NEXT: v_mov_b32_e32 v4, s3
+; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX11-NEXT: scratch_store_b32 off, v0, off
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_clause 0x4
+; GFX11-NEXT: scratch_store_b32 off, v0, off offset:16
+; GFX11-NEXT: scratch_store_b16 off, v1, off offset:4
+; GFX11-NEXT: scratch_store_b16 off, v2, off offset:20
+; GFX11-NEXT: scratch_store_b32 off, v3, off offset:8
+; GFX11-NEXT: scratch_store_b16 off, v4, off offset:12
+; GFX11-NEXT: s_clause 0xb
+; GFX11-NEXT: scratch_load_u8 v0, off, off offset:21
+; GFX11-NEXT: scratch_load_u8 v1, off, off offset:20
+; GFX11-NEXT: scratch_load_u8 v2, off, off offset:19
+; GFX11-NEXT: scratch_load_u8 v3, off, off offset:18
+; GFX11-NEXT: scratch_load_u8 v4, off, off offset:11
+; GFX11-NEXT: scratch_load_u8 v5, off, off offset:12
+; GFX11-NEXT: scratch_load_u8 v6, off, off offset:5
+; GFX11-NEXT: scratch_load_u8 v7, off, off offset:4
+; GFX11-NEXT: scratch_load_u8 v8, off, off offset:3
+; GFX11-NEXT: scratch_load_u8 v9, off, off offset:13
+; GFX11-NEXT: scratch_load_u8 v10, off, off offset:2
+; GFX11-NEXT: scratch_load_u8 v11, off, off offset:10
+; GFX11-NEXT: s_waitcnt vmcnt(11)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_waitcnt vmcnt(10)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX11-NEXT: s_waitcnt vmcnt(9)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or3_b32 v0, v1, v2, v0
+; GFX11-NEXT: s_waitcnt vmcnt(8)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: v_lshl_or_b32 v3, v5, 8, v4
+; GFX11-NEXT: s_waitcnt vmcnt(5)
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 8, v7
+; GFX11-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
+; GFX11-NEXT: v_and_or_b32 v1, 0xffff, s4, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_hi_u32 v1, 0xaaaaaab, v1
+; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: v_lshl_or_b32 v3, v9, 24, v3
+; GFX11-NEXT: v_mul_u32_u24_e32 v0, 24, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v0, v2, v0
+; GFX11-NEXT: v_mul_u32_u24_e32 v1, 24, v1
+; GFX11-NEXT: v_or3_b32 v2, v5, v8, v4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v10
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshl_or_b32 v5, v11, 24, s1
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 8, v0
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, s4, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_alignbit_b32 v0, v2, v3, v0.l
+; GFX11-NEXT: v_and_or_b32 v2, 0xffff, s0, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 8, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-NEXT: v_alignbit_b32 v1, v2, v5, v1.l
+; GFX11-NEXT: s_clause 0x4
+; GFX11-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
+; GFX11-NEXT: scratch_store_b8 off, v3, off offset:28
+; GFX11-NEXT: scratch_store_b8 off, v0, off offset:27
+; GFX11-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
+; GFX11-NEXT: scratch_store_b16 off, v1, off offset:24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: scratch_load_b32 v0, off, off offset:24
+; GFX11-NEXT: scratch_load_u16 v1, off, off offset:28
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i24:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_getpc_b64 s[8:9]
-; GFX9-NEXT: s_mov_b32 s8, s0
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: s_mov_b32 s1, 0xaaaaaab
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_u32 s8, s8, s6
-; GFX9-NEXT: s_addc_u32 s9, s9, 0
-; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:4
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:20
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:16
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:8
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:12
-; GFX9-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:5
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:21
-; GFX9-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:13
-; GFX9-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:12
-; GFX9-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:4
-; GFX9-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:20
-; GFX9-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:19
-; GFX9-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:11
-; GFX9-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:2
-; GFX9-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:18
-; GFX9-NEXT: buffer_load_ubyte v10, off, s[8:11], 0 offset:10
-; GFX9-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:3
-; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX9-NEXT: s_and_b32 s3, s4, 0xffff
-; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX9-NEXT: s_lshl_b32 s2, s2, 8
-; GFX9-NEXT: s_waitcnt vmcnt(11)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_waitcnt vmcnt(10)
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_or3_b32 v1, v5, v6, v1
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 8, v7
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v7, v8, 16, s0
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, s3
-; GFX9-NEXT: v_mul_hi_u32 v1, v1, s1
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or3_b32 v0, v4, v11, v0
-; GFX9-NEXT: v_mul_hi_u32 v4, v8, s1
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 24, v3
-; GFX9-NEXT: v_mul_u32_u24_e32 v1, 24, v1
-; GFX9-NEXT: v_mul_u32_u24_e32 v3, 24, v4
-; GFX9-NEXT: v_sub_u32_e32 v3, s4, v3
-; GFX9-NEXT: v_sub_u32_e32 v1, v6, v1
-; GFX9-NEXT: v_lshl_or_b32 v9, v10, 24, s2
-; GFX9-NEXT: v_add_u32_e32 v3, 8, v3
-; GFX9-NEXT: v_add_u32_e32 v1, 8, v1
-; GFX9-NEXT: v_alignbit_b32 v3, v7, v9, v3
-; GFX9-NEXT: v_alignbit_b32 v0, v0, v2, v1
-; GFX9-NEXT: buffer_store_byte_d16_hi v3, off, s[8:11], 0 offset:26
-; GFX9-NEXT: buffer_store_short v3, off, s[8:11], 0 offset:24
-; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[8:11], 0 offset:29
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX9-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:27
-; GFX9-NEXT: buffer_store_byte v1, off, s[8:11], 0 offset:28
-; GFX9-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-NEXT: s_lshr_b32 s9, s1, 8
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_lshr_b32 s6, s0, 8
+; GFX9-NEXT: s_lshr_b32 s8, s0, 24
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_or_b32 s1, s8, s1
+; GFX9-NEXT: s_lshr_b32 s8, s2, 8
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s6, s6, 8
+; GFX9-NEXT: s_and_b32 s8, s8, 0xff
+; GFX9-NEXT: s_or_b32 s0, s0, s6
+; GFX9-NEXT: s_and_b32 s6, s7, 0xff
+; GFX9-NEXT: s_and_b32 s7, s9, 0xff
+; GFX9-NEXT: s_lshr_b32 s9, s2, 16
+; GFX9-NEXT: s_lshr_b32 s10, s2, 24
+; GFX9-NEXT: s_and_b32 s2, s2, 0xff
+; GFX9-NEXT: s_lshl_b32 s8, s8, 8
+; GFX9-NEXT: s_or_b32 s2, s2, s8
+; GFX9-NEXT: s_and_b32 s8, s9, 0xff
+; GFX9-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX9-NEXT: s_lshr_b32 s11, s3, 8
+; GFX9-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX9-NEXT: s_lshl_b32 s8, s8, 16
+; GFX9-NEXT: s_and_b32 s3, s3, 0xff
+; GFX9-NEXT: s_or_b32 s2, s2, s8
+; GFX9-NEXT: s_lshl_b32 s3, s3, 8
+; GFX9-NEXT: s_and_b32 s8, s11, 0xff
+; GFX9-NEXT: s_or_b32 s3, s10, s3
+; GFX9-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX9-NEXT: s_lshl_b32 s8, s8, 16
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX9-NEXT: s_or_b32 s3, s3, s8
+; GFX9-NEXT: s_lshr_b32 s8, s4, 8
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: s_and_b32 s8, s8, 0xff
+; GFX9-NEXT: s_lshr_b32 s9, s4, 16
+; GFX9-NEXT: s_lshr_b32 s10, s4, 24
+; GFX9-NEXT: s_and_b32 s4, s4, 0xff
+; GFX9-NEXT: s_lshl_b32 s8, s8, 8
+; GFX9-NEXT: s_or_b32 s4, s4, s8
+; GFX9-NEXT: s_and_b32 s8, s9, 0xff
+; GFX9-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: s_lshr_b32 s11, s5, 8
+; GFX9-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX9-NEXT: s_lshl_b32 s8, s8, 16
+; GFX9-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: s_or_b32 s4, s4, s8
+; GFX9-NEXT: s_lshl_b32 s5, s5, 8
+; GFX9-NEXT: s_and_b32 s8, s11, 0xff
+; GFX9-NEXT: s_or_b32 s5, s10, s5
+; GFX9-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX9-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX9-NEXT: s_lshl_b32 s8, s8, 16
+; GFX9-NEXT: s_or_b32 s5, s5, s8
+; GFX9-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9-NEXT: s_mul_i32 s9, s8, 0xffffffe8
+; GFX9-NEXT: s_mul_hi_u32 s9, s8, s9
+; GFX9-NEXT: s_add_i32 s8, s8, s9
+; GFX9-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX9-NEXT: s_mulk_i32 s9, 0xffe8
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX9-NEXT: s_add_i32 s4, s4, s9
+; GFX9-NEXT: s_cmp_ge_u32 s4, 24
+; GFX9-NEXT: s_cselect_b32 s9, 1, 0
+; GFX9-NEXT: s_sub_i32 s10, s4, 24
+; GFX9-NEXT: s_cmp_lg_u32 s9, 0
+; GFX9-NEXT: s_cselect_b32 s4, s10, s4
+; GFX9-NEXT: s_cmp_ge_u32 s4, 24
+; GFX9-NEXT: s_cselect_b32 s9, 1, 0
+; GFX9-NEXT: s_sub_i32 s10, s4, 24
+; GFX9-NEXT: s_cmp_lg_u32 s9, 0
+; GFX9-NEXT: s_cselect_b32 s4, s10, s4
+; GFX9-NEXT: s_lshl_b32 s6, s6, 17
+; GFX9-NEXT: s_lshl_b32 s0, s0, 1
+; GFX9-NEXT: s_sub_i32 s9, 23, s4
+; GFX9-NEXT: s_or_b32 s0, s6, s0
+; GFX9-NEXT: s_lshl_b32 s0, s0, s9
+; GFX9-NEXT: s_lshr_b32 s2, s2, s4
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_mul_hi_u32 s2, s5, s8
+; GFX9-NEXT: s_mulk_i32 s2, 0xffe8
+; GFX9-NEXT: s_add_i32 s5, s5, s2
+; GFX9-NEXT: s_cmp_ge_u32 s5, 24
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_sub_i32 s4, s5, 24
+; GFX9-NEXT: s_cmp_lg_u32 s2, 0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s5
+; GFX9-NEXT: s_cmp_ge_u32 s2, 24
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_sub_i32 s5, s2, 24
+; GFX9-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-NEXT: s_cselect_b32 s2, s5, s2
+; GFX9-NEXT: s_lshl_b32 s5, s7, 17
+; GFX9-NEXT: s_lshl_b32 s1, s1, 1
+; GFX9-NEXT: s_sub_i32 s4, 23, s2
+; GFX9-NEXT: s_or_b32 s1, s5, s1
+; GFX9-NEXT: s_lshl_b32 s1, s1, s4
+; GFX9-NEXT: s_lshr_b32 s2, s3, s2
+; GFX9-NEXT: s_bfe_u32 s3, s0, 0x80008
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_and_b32 s2, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s3, s3, 8
+; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX9-NEXT: s_or_b32 s2, s2, s3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: s_or_b32 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_bfe_u32 s2, s1, 0x80008
+; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_or_b32 s1, s2, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_v2i24:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_getpc_b64 s[8:9]
-; GFX10-NEXT: s_mov_b32 s8, s0
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s5
-; GFX10-NEXT: v_mov_b32_e32 v3, s4
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bitset0_b32 s11, 21
-; GFX10-NEXT: s_add_u32 s8, s8, s6
-; GFX10-NEXT: s_addc_u32 s9, s9, 0
-; GFX10-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX10-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s3
-; GFX10-NEXT: buffer_store_short v1, off, s[8:11], 0 offset:4
-; GFX10-NEXT: buffer_store_short v2, off, s[8:11], 0 offset:20
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX10-NEXT: s_lshr_b32 s9, s1, 8
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshr_b32 s6, s0, 8
+; GFX10-NEXT: s_lshr_b32 s8, s0, 24
; GFX10-NEXT: s_lshl_b32 s1, s1, 8
-; GFX10-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:12
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:21
-; GFX10-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:20
-; GFX10-NEXT: buffer_store_dword v3, off, s[8:11], 0 offset:16
-; GFX10-NEXT: buffer_store_dword v1, off, s[8:11], 0 offset:8
-; GFX10-NEXT: s_clause 0x9
-; GFX10-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:18
-; GFX10-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:19
-; GFX10-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:11
-; GFX10-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:13
-; GFX10-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:12
-; GFX10-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:5
-; GFX10-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:4
-; GFX10-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:2
-; GFX10-NEXT: buffer_load_ubyte v10, off, s[8:11], 0 offset:3
-; GFX10-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:10
-; GFX10-NEXT: s_waitcnt vmcnt(9)
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-NEXT: v_and_or_b32 v1, 0xffff, s4, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT: v_mul_hi_u32 v1, 0xaaaaaab, v1
-; GFX10-NEXT: s_waitcnt vmcnt(8)
-; GFX10-NEXT: v_or3_b32 v0, v2, v3, v0
-; GFX10-NEXT: s_waitcnt vmcnt(5)
-; GFX10-NEXT: v_lshl_or_b32 v2, v6, 8, v4
-; GFX10-NEXT: s_waitcnt vmcnt(4)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX10-NEXT: s_waitcnt vmcnt(3)
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 8, v8
-; GFX10-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX10-NEXT: v_mul_u32_u24_e32 v1, 24, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_or3_b32 v4, v6, v10, v4
-; GFX10-NEXT: v_sub_nc_u32_e32 v1, s4, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v5, 24, v2
-; GFX10-NEXT: v_mul_u32_u24_e32 v0, 24, v0
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 8, v1
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, v3, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v9
-; GFX10-NEXT: v_add_nc_u32_e32 v0, 8, v0
-; GFX10-NEXT: v_and_or_b32 v3, 0xffff, s0, v3
-; GFX10-NEXT: v_alignbit_b32 v0, v4, v2, v0
-; GFX10-NEXT: v_alignbit_b32 v1, v3, v5, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: buffer_store_byte_d16_hi v0, off, s[8:11], 0 offset:29
-; GFX10-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:27
-; GFX10-NEXT: buffer_store_byte_d16_hi v1, off, s[8:11], 0 offset:26
-; GFX10-NEXT: buffer_store_short v1, off, s[8:11], 0 offset:24
-; GFX10-NEXT: buffer_store_byte v2, off, s[8:11], 0 offset:28
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX10-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT: s_and_b32 s6, s6, 0xff
+; GFX10-NEXT: s_or_b32 s1, s8, s1
+; GFX10-NEXT: s_lshr_b32 s8, s2, 8
+; GFX10-NEXT: s_lshr_b32 s7, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s6, s6, 8
+; GFX10-NEXT: s_and_b32 s8, s8, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s6
+; GFX10-NEXT: s_and_b32 s6, s7, 0xff
+; GFX10-NEXT: s_and_b32 s7, s9, 0xff
+; GFX10-NEXT: s_lshr_b32 s9, s2, 16
+; GFX10-NEXT: s_lshr_b32 s10, s2, 24
+; GFX10-NEXT: s_and_b32 s2, s2, 0xff
+; GFX10-NEXT: s_lshl_b32 s8, s8, 8
+; GFX10-NEXT: s_lshr_b32 s11, s3, 8
+; GFX10-NEXT: s_or_b32 s2, s2, s8
+; GFX10-NEXT: s_and_b32 s8, s9, 0xff
+; GFX10-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX10-NEXT: s_lshl_b32 s8, s8, 16
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_and_b32 s9, s11, 0xff
+; GFX10-NEXT: s_or_b32 s3, s10, s3
+; GFX10-NEXT: s_and_b32 s9, 0xffff, s9
+; GFX10-NEXT: s_or_b32 s2, s2, s8
+; GFX10-NEXT: s_lshr_b32 s8, s4, 8
+; GFX10-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX10-NEXT: s_lshl_b32 s9, s9, 16
+; GFX10-NEXT: s_and_b32 s8, s8, 0xff
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT: s_or_b32 s3, s3, s9
+; GFX10-NEXT: s_lshr_b32 s9, s4, 16
+; GFX10-NEXT: s_lshr_b32 s10, s4, 24
+; GFX10-NEXT: s_and_b32 s4, s4, 0xff
+; GFX10-NEXT: s_lshl_b32 s8, s8, 8
+; GFX10-NEXT: s_lshr_b32 s11, s5, 8
+; GFX10-NEXT: s_or_b32 s4, s4, s8
+; GFX10-NEXT: s_and_b32 s8, s9, 0xff
+; GFX10-NEXT: v_readfirstlane_b32 s9, v0
+; GFX10-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX10-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX10-NEXT: s_lshl_b32 s8, s8, 16
+; GFX10-NEXT: s_and_b32 s5, s5, 0xff
+; GFX10-NEXT: s_or_b32 s4, s4, s8
+; GFX10-NEXT: s_mul_i32 s8, s9, 0xffffffe8
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_mul_hi_u32 s8, s9, s8
+; GFX10-NEXT: s_or_b32 s5, s10, s5
+; GFX10-NEXT: s_add_i32 s9, s9, s8
+; GFX10-NEXT: s_and_b32 s10, s11, 0xff
+; GFX10-NEXT: s_mul_hi_u32 s8, s4, s9
+; GFX10-NEXT: s_and_b32 s10, 0xffff, s10
+; GFX10-NEXT: s_mulk_i32 s8, 0xffe8
+; GFX10-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX10-NEXT: s_lshl_b32 s10, s10, 16
+; GFX10-NEXT: s_add_i32 s4, s4, s8
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX10-NEXT: s_or_b32 s5, s5, s10
+; GFX10-NEXT: s_cmp_ge_u32 s4, 24
+; GFX10-NEXT: s_cselect_b32 s8, 1, 0
+; GFX10-NEXT: s_sub_i32 s10, s4, 24
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: s_cselect_b32 s4, s10, s4
+; GFX10-NEXT: s_cmp_ge_u32 s4, 24
+; GFX10-NEXT: s_cselect_b32 s8, 1, 0
+; GFX10-NEXT: s_sub_i32 s10, s4, 24
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: s_cselect_b32 s4, s10, s4
+; GFX10-NEXT: s_lshl_b32 s6, s6, 17
+; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_sub_i32 s8, 23, s4
+; GFX10-NEXT: s_or_b32 s0, s6, s0
+; GFX10-NEXT: s_mul_hi_u32 s6, s5, s9
+; GFX10-NEXT: s_lshl_b32 s0, s0, s8
+; GFX10-NEXT: s_mulk_i32 s6, 0xffe8
+; GFX10-NEXT: s_lshr_b32 s2, s2, s4
+; GFX10-NEXT: s_add_i32 s5, s5, s6
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_cmp_ge_u32 s5, 24
+; GFX10-NEXT: s_cselect_b32 s2, 1, 0
+; GFX10-NEXT: s_sub_i32 s4, s5, 24
+; GFX10-NEXT: s_cmp_lg_u32 s2, 0
+; GFX10-NEXT: s_cselect_b32 s2, s4, s5
+; GFX10-NEXT: s_cmp_ge_u32 s2, 24
+; GFX10-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-NEXT: s_sub_i32 s5, s2, 24
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: s_cselect_b32 s2, s5, s2
+; GFX10-NEXT: s_lshl_b32 s4, s7, 17
+; GFX10-NEXT: s_lshl_b32 s1, s1, 1
+; GFX10-NEXT: s_sub_i32 s5, 23, s2
+; GFX10-NEXT: s_or_b32 s1, s4, s1
+; GFX10-NEXT: s_lshr_b32 s2, s3, s2
+; GFX10-NEXT: s_lshl_b32 s1, s1, s5
+; GFX10-NEXT: s_and_b32 s3, s0, 0xff
+; GFX10-NEXT: s_or_b32 s1, s1, s2
+; GFX10-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX10-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_lshl_b32 s0, s0, 16
+; GFX10-NEXT: s_or_b32 s2, s3, s2
+; GFX10-NEXT: s_and_b32 s3, s1, 0xff
+; GFX10-NEXT: s_or_b32 s0, s2, s0
+; GFX10-NEXT: s_lshl_b32 s2, s3, 24
+; GFX10-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX10-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_fshr_v2i24:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v0, off
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s2
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_clause 0x4
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v0, off offset:16
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v1, off offset:4
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v2, off offset:20
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v3, off offset:8
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v4, off offset:12
-; GFX11-TRUE16-NEXT: s_clause 0xb
-; GFX11-TRUE16-NEXT: scratch_load_u8 v0, off, off offset:21
-; GFX11-TRUE16-NEXT: scratch_load_u8 v1, off, off offset:20
-; GFX11-TRUE16-NEXT: scratch_load_u8 v2, off, off offset:19
-; GFX11-TRUE16-NEXT: scratch_load_u8 v3, off, off offset:18
-; GFX11-TRUE16-NEXT: scratch_load_u8 v4, off, off offset:11
-; GFX11-TRUE16-NEXT: scratch_load_u8 v5, off, off offset:12
-; GFX11-TRUE16-NEXT: scratch_load_u8 v6, off, off offset:5
-; GFX11-TRUE16-NEXT: scratch_load_u8 v7, off, off offset:4
-; GFX11-TRUE16-NEXT: scratch_load_u8 v8, off, off offset:3
-; GFX11-TRUE16-NEXT: scratch_load_u8 v9, off, off offset:13
-; GFX11-TRUE16-NEXT: scratch_load_u8 v10, off, off offset:2
-; GFX11-TRUE16-NEXT: scratch_load_u8 v11, off, off offset:10
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or3_b32 v0, v1, v2, v0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v5, 8, v4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 8, v7
-; GFX11-TRUE16-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff, s4, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mul_hi_u32 v1, 0xaaaaaab, v1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v9, 24, v3
-; GFX11-TRUE16-NEXT: v_mul_u32_u24_e32 v0, 24, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_mul_u32_u24_e32 v1, 24, v1
-; GFX11-TRUE16-NEXT: v_or3_b32 v2, v5, v8, v4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v10
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v1, s4, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v2, v3, v0.l
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff, s0, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v5, v1.l
-; GFX11-TRUE16-NEXT: s_clause 0x4
-; GFX11-TRUE16-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
-; GFX11-TRUE16-NEXT: scratch_store_b8 off, v3, off offset:28
-; GFX11-TRUE16-NEXT: scratch_store_b8 off, v0, off offset:27
-; GFX11-TRUE16-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
-; GFX11-TRUE16-NEXT: scratch_store_b16 off, v1, off offset:24
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, off offset:24
-; GFX11-TRUE16-NEXT: scratch_load_u16 v1, off, off offset:28
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: s_fshr_v2i24:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v0, off
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s2
+; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s0, 24
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-FAKE16-NEXT: s_clause 0x4
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v0, off offset:16
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v1, off offset:4
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v2, off offset:20
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v3, off offset:8
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v4, off offset:12
-; GFX11-FAKE16-NEXT: s_clause 0xb
-; GFX11-FAKE16-NEXT: scratch_load_u8 v0, off, off offset:21
-; GFX11-FAKE16-NEXT: scratch_load_u8 v1, off, off offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u8 v2, off, off offset:19
-; GFX11-FAKE16-NEXT: scratch_load_u8 v3, off, off offset:18
-; GFX11-FAKE16-NEXT: scratch_load_u8 v4, off, off offset:11
-; GFX11-FAKE16-NEXT: scratch_load_u8 v5, off, off offset:12
-; GFX11-FAKE16-NEXT: scratch_load_u8 v6, off, off offset:5
-; GFX11-FAKE16-NEXT: scratch_load_u8 v7, off, off offset:4
-; GFX11-FAKE16-NEXT: scratch_load_u8 v8, off, off offset:3
-; GFX11-FAKE16-NEXT: scratch_load_u8 v9, off, off offset:13
-; GFX11-FAKE16-NEXT: scratch_load_u8 v10, off, off offset:2
-; GFX11-FAKE16-NEXT: scratch_load_u8 v11, off, off offset:10
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or3_b32 v0, v1, v2, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v5, 8, v4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(4)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 8, v7
-; GFX11-FAKE16-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xffff, s4, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mul_hi_u32 v1, 0xaaaaaab, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v9, 24, v3
-; GFX11-FAKE16-NEXT: v_mul_u32_u24_e32 v0, 24, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_sub_nc_u32_e32 v0, v2, v0
-; GFX11-FAKE16-NEXT: v_mul_u32_u24_e32 v1, 24, v1
-; GFX11-FAKE16-NEXT: v_or3_b32 v2, v5, v8, v4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v10
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_sub_nc_u32_e32 v1, s4, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v2, v3, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff, s0, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 8, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v5, v1
-; GFX11-FAKE16-NEXT: s_clause 0x4
-; GFX11-FAKE16-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
-; GFX11-FAKE16-NEXT: scratch_store_b8 off, v3, off offset:28
-; GFX11-FAKE16-NEXT: scratch_store_b8 off, v0, off offset:27
-; GFX11-FAKE16-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
-; GFX11-FAKE16-NEXT: scratch_store_b16 off, v1, off offset:24
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, off offset:24
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, off offset:28
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s8, s1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s9, 0xff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s2, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s2, 24
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s8, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s3, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s9, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s8, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s9, s11, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s10, s3
+; GFX11-FAKE16-NEXT: s_and_b32 s9, 0xffff, s9
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s9, s9, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, 0xff
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s3, s9
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s4, 24
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s8, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s5, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s4, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s9, 0xff
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s8, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s5, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s4, s8
+; GFX11-FAKE16-NEXT: s_mul_i32 s8, s9, 0xffffffe8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s8, s9, s8
+; GFX11-FAKE16-NEXT: s_or_b32 s5, s10, s5
+; GFX11-FAKE16-NEXT: s_add_i32 s9, s9, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s10, s11, 0xff
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s8, s4, s9
+; GFX11-FAKE16-NEXT: s_and_b32 s10, 0xffff, s10
+; GFX11-FAKE16-NEXT: s_mulk_i32 s8, 0xffe8
+; GFX11-FAKE16-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX11-FAKE16-NEXT: s_lshl_b32 s10, s10, 16
+; GFX11-FAKE16-NEXT: s_add_i32 s4, s4, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX11-FAKE16-NEXT: s_or_b32 s5, s5, s10
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s10, s4, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, s10, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s10, s4, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, s10, s4
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 17
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-FAKE16-NEXT: s_sub_i32 s8, 23, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s6, s0
+; GFX11-FAKE16-NEXT: s_mul_hi_u32 s6, s5, s9
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
+; GFX11-FAKE16-NEXT: s_mulk_i32 s6, 0xffe8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s4
+; GFX11-FAKE16-NEXT: s_add_i32 s5, s5, s6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s5, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s4, s5, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, s4, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s5, s2, 24
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, s5, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s7, 17
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 1
+; GFX11-FAKE16-NEXT: s_sub_i32 s5, 23, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s4, s1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s3, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s0, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s3, s2
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s3, 24
+; GFX11-FAKE16-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s3
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i48 %lhs.arg to <2 x i24>
%rhs = bitcast i48 %rhs.arg to <2 x i24>
@@ -2286,6 +2542,59 @@ define <2 x i24> @v_fshr_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v2i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_dual_mul_f32 v6, 0x4f7ffffe, v6 :: v_dual_lshlrev_b32 v1, 1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
+; GFX11-NEXT: v_readfirstlane_b32 s0, v6
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
+; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v6, v2
+; GFX11-NEXT: v_lshl_or_b32 v1, v1, v4, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2331,7 +2640,6 @@ define <2 x i24> @v_fshr_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v2, v2, v3
; GFX9-NEXT: v_lshl_or_b32 v1, v1, v4, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2375,82 +2683,27 @@ define <2 x i24> @v_fshr_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v4, v3
; GFX10-NEXT: v_lshl_or_b32 v0, v0, v6, v2
; GFX10-NEXT: v_lshl_or_b32 v1, v1, v5, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_v2i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_dual_mul_f32 v6, 0x4f7ffffe, v6 :: v_dual_lshlrev_b32 v1, 1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
-; GFX11-NEXT: v_readfirstlane_b32 s0, v6
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
-; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
-; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v6, v2
-; GFX11-NEXT: v_lshl_or_b32 v1, v1, v4, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i24> @llvm.fshr.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
ret <2 x i24> %result
}
define amdgpu_ps i32 @s_fshr_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GFX6-LABEL: s_fshr_i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshr_i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshr_i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -2458,22 +2711,12 @@ define amdgpu_ps i32 @s_fshr_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, v0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_fshr_i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: s_fshr_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2486,77 +2729,59 @@ define amdgpu_ps i32 @s_fshr_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
}
define amdgpu_ps i32 @s_fshr_i32_5(i32 inreg %lhs, i32 inreg %rhs) {
-; GFX6-LABEL: s_fshr_i32_5:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 5
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshr_i32_5:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 5
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshr_i32_5:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 5
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i32_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i32_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 5
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i32_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 5
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i32_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 5)
ret i32 %result
}
define amdgpu_ps i32 @s_fshr_i32_8(i32 inreg %lhs, i32 inreg %rhs) {
-; GFX6-LABEL: s_fshr_i32_8:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 8
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshr_i32_8:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 8
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshr_i32_8:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 8
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i32_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i32_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 8
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i32_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 8
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i32_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 8)
ret i32 %result
}
@@ -2568,12 +2793,11 @@ define i32 @v_fshr_i32(i32 %lhs, i32 %rhs, i32 %amt) {
; GCN-NEXT: v_alignbit_b32 v0, v0, v1, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_fshr_i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
+; GFX11-LABEL: v_fshr_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-FAKE16-LABEL: v_fshr_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2616,34 +2840,25 @@ define i32 @v_fshr_i32_8(i32 %lhs, i32 %rhs) {
}
define amdgpu_ps float @v_fshr_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt) {
-; GFX6-LABEL: v_fshr_i32_ssv:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v1, v0
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: v_fshr_i32_ssv:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v1, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: v_fshr_i32_ssv:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v1, v0
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i32_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i32_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i32_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshr_i32_ssv:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshr_i32_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, v0
@@ -2654,36 +2869,27 @@ define amdgpu_ps float @v_fshr_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
}
define amdgpu_ps float @v_fshr_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt) {
-; GFX6-LABEL: v_fshr_i32_svs:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: v_fshr_i32_svs:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: v_fshr_i32_svs:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i32_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i32_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i32_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshr_i32_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshr_i32_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
@@ -2694,40 +2900,30 @@ define amdgpu_ps float @v_fshr_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
}
define amdgpu_ps float @v_fshr_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GFX6-LABEL: v_fshr_i32_vss:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: v_fshr_i32_vss:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: v_fshr_i32_vss:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i32_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i32_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i32_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshr_i32_vss:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshr_i32_vss:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2747,13 +2943,12 @@ define <2 x i32> @v_fshr_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
; GCN-NEXT: v_alignbit_b32 v1, v1, v3, v5
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_fshr_v2i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
+; GFX11-LABEL: v_fshr_v2i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
+; GFX11-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-FAKE16-LABEL: v_fshr_v2i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2773,14 +2968,13 @@ define <3 x i32> @v_fshr_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
; GCN-NEXT: v_alignbit_b32 v2, v2, v5, v8
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_fshr_v3i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
+; GFX11-LABEL: v_fshr_v3i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
+; GFX11-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
+; GFX11-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-FAKE16-LABEL: v_fshr_v3i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2802,15 +2996,14 @@ define <4 x i32> @v_fshr_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
; GCN-NEXT: v_alignbit_b32 v3, v3, v7, v11
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_fshr_v4i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
+; GFX11-LABEL: v_fshr_v4i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
+; GFX11-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
+; GFX11-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
+; GFX11-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-FAKE16-LABEL: v_fshr_v4i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2846,6 +3039,17 @@ define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s3, s2, 15
+; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s3, s2, 15
@@ -2856,7 +3060,6 @@ define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s3
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s3, s2, 15
@@ -2867,18 +3070,6 @@ define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s3, s2, 15
-; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 %amt)
ret i16 %result
}
@@ -2899,6 +3090,14 @@ define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i16_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 12
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -2906,7 +3105,6 @@ define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -2914,15 +3112,6 @@ define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 4
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i16_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 12
-; GFX11-NEXT: s_lshr_b32 s1, s1, 4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 4)
ret i16 %result
}
@@ -2943,6 +3132,14 @@ define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_i16_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 11
+; GFX11-NEXT: s_lshr_b32 s1, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -2950,7 +3147,6 @@ define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 5
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -2958,15 +3154,6 @@ define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 5
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i16_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 11
-; GFX11-NEXT: s_lshr_b32 s1, s1, 5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 5)
ret i16 %result
}
@@ -2997,6 +3184,19 @@ define i16 @v_fshr_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b16 v0.h, v2.l, -1
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-NEXT: v_and_b16 v1.h, v2.l, 15
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3008,7 +3208,6 @@ define i16 @v_fshr_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, v3, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3020,21 +3219,6 @@ define i16 @v_fshr_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX10-NEXT: v_lshlrev_b16 v0, v3, v0
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, -1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3069,6 +3253,14 @@ define i16 @v_fshr_i16_4(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i16_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 12, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 4, v1.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3076,7 +3268,6 @@ define i16 @v_fshr_i16_4(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 4, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3084,16 +3275,6 @@ define i16 @v_fshr_i16_4(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 4, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i16_4:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 12, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 4, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_i16_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3123,6 +3304,14 @@ define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i16_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 11, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v0.h, 5, v1.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3130,7 +3319,6 @@ define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 5, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3138,16 +3326,6 @@ define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 5, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i16_5:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 11, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 5, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-FAKE16-LABEL: v_fshr_i16_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3163,56 +3341,66 @@ define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
define amdgpu_ps half @v_fshr_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt) {
; GFX6-LABEL: v_fshr_i16_ssv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: v_or_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v1, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 15, v0
+; GFX6-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: v_lshl_b32_e32 v0, s0, v0
+; GFX6-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX6-NEXT: v_lshr_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i16_ssv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v0
+; GFX8-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: v_lshlrev_b16_e64 v1, v1, s0
-; GFX8-NEXT: v_lshrrev_b16_e64 v0, v0, s1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_lshlrev_b16_e64 v0, v0, s0
+; GFX8-NEXT: v_lshrrev_b16_e64 v1, v1, s1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i16_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b16 v0.h, v0.l, -1
+; GFX11-NEXT: v_and_b16 v0.l, v0.l, 15
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-NEXT: v_lshrrev_b16 v0.l, v0.l, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b16 v0.h, v0.h, s0
+; GFX11-NEXT: v_or_b16 v0.l, v0.h, v0.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i16_ssv:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 15, v0
+; GFX9-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: v_lshlrev_b16_e64 v1, v1, s0
-; GFX9-NEXT: v_lshrrev_b16_e64 v0, v0, s1
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_lshlrev_b16_e64 v0, v0, s0
+; GFX9-NEXT: v_lshrrev_b16_e64 v1, v1, s1
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT: v_and_b32_e32 v0, 15, v0
; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: v_and_b32_e32 v1, 15, v1
; GFX10-NEXT: v_lshrrev_b16 v0, v0, s1
; GFX10-NEXT: v_lshlrev_b16 v1, v1, s0
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshr_i16_ssv:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v0.l, -1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.l, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshr_i16_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 15, v0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 15, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, v0, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, v1, s0
@@ -3226,32 +3414,45 @@ define amdgpu_ps half @v_fshr_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
define amdgpu_ps half @v_fshr_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt) {
; GFX6-LABEL: v_fshr_i16_svs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_or_b32 s1, s1, 16
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT: s_and_b32 s2, s1, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, s1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s2, v0
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i16_svs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
+; GFX8-NEXT: s_and_b32 s2, s1, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: s_and_b32 s1, s1, 15
-; GFX8-NEXT: s_lshl_b32 s0, s0, s2
-; GFX8-NEXT: v_lshrrev_b16_e32 v0, s1, v0
+; GFX8-NEXT: s_lshl_b32 s0, s0, s1
+; GFX8-NEXT: v_lshrrev_b16_e32 v0, s2, v0
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i16_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s2, s1, 15
+; GFX11-NEXT: s_and_not1_b32 s1, 15, s1
+; GFX11-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i16_svs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_andn2_b32 s2, 15, s1
+; GFX9-NEXT: s_and_b32 s2, s1, 15
+; GFX9-NEXT: s_andn2_b32 s1, 15, s1
; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: s_and_b32 s1, s1, 15
-; GFX9-NEXT: s_lshl_b32 s0, s0, s2
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, s1, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, s1
+; GFX9-NEXT: v_lshrrev_b16_e32 v0, s2, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s2, s1, 15
@@ -3261,19 +3462,6 @@ define amdgpu_ps half @v_fshr_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX10-NEXT: s_lshl_b32 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshr_i16_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, 15
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s1, 15, s1
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshr_i16_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_and_b32 s2, s1, 15
@@ -3293,64 +3481,64 @@ define amdgpu_ps half @v_fshr_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt) {
; GFX6-LABEL: v_fshr_i16_vss:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_or_b32 s1, s1, 16
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_alignbit_b32 v0, v0, s0, v1
+; GFX6-NEXT: s_and_b32 s2, s1, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX6-NEXT: s_lshr_b32 s0, s0, s2
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i16_vss:
; GFX8: ; %bb.0:
+; GFX8-NEXT: s_and_b32 s2, s1, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
; GFX8-NEXT: v_lshlrev_b16_e32 v0, 1, v0
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX8-NEXT: s_and_b32 s1, s1, 15
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, s2, v0
-; GFX8-NEXT: s_lshr_b32 s0, s0, s1
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, s1, v0
+; GFX8-NEXT: s_lshr_b32 s0, s0, s2
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i16_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-NEXT: s_and_not1_b32 s2, 15, s1
+; GFX11-NEXT: s_and_b32 s1, s1, 15
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
+; GFX11-NEXT: s_lshr_b32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b16 v0.l, v0.l, s0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i16_vss:
; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s2, s1, 15
+; GFX9-NEXT: s_andn2_b32 s1, 15, s1
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 1, v0
-; GFX9-NEXT: s_andn2_b32 s2, 15, s1
; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX9-NEXT: s_and_b32 s1, s1, 15
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, s2, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, s1
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, s1, v0
+; GFX9-NEXT: s_lshr_b32 s0, s0, s2
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
; GFX10-NEXT: s_andn2_b32 s2, 15, s1
-; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10-NEXT: s_and_b32 s1, s1, 15
+; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10-NEXT: s_lshr_b32 s0, s0, s1
; GFX10-NEXT: v_lshlrev_b16 v0, s2, v0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: v_fshr_i16_vss:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
-; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: v_fshr_i16_vss:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 1, v0
; GFX11-FAKE16-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 15
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, s1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, s2, v0
@@ -3403,6 +3591,28 @@ define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_and_b32 s4, s2, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s2, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s2, s3, s5
+; GFX11-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_lshr_b32 s5, s4, 16
+; GFX11-NEXT: s_lshr_b32 s3, s3, s4
+; GFX11-NEXT: s_lshr_b32 s1, s1, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s4, s0, 16
@@ -3424,7 +3634,6 @@ define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s3, s0, 16
@@ -3446,29 +3655,6 @@ define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s3, s3, 1
-; GFX11-NEXT: s_and_b32 s4, s2, 0xf000f
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
-; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
-; GFX11-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-NEXT: s_lshr_b32 s5, s2, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshl_b32 s2, s3, s5
-; GFX11-NEXT: s_and_b32 s3, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_lshr_b32 s5, s4, 16
-; GFX11-NEXT: s_lshr_b32 s3, s3, s4
-; GFX11-NEXT: s_lshr_b32 s1, s1, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s3, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to i32
ret i32 %cast
@@ -3509,6 +3695,19 @@ define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v3, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3520,7 +3719,6 @@ define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v3, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3532,20 +3730,6 @@ define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX10-NEXT: v_pk_lshlrev_b16 v0, v3, v0
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, v2, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v3, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
ret <2 x i16> %result
}
@@ -3575,6 +3759,14 @@ define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v2i16_4_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x8000c, v0
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x80004, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i16_4_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3584,7 +3776,6 @@ define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v2i16_4_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3592,15 +3783,6 @@ define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX10-NEXT: v_pk_lshrrev_b16 v1, 0x80004, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_v2i16_4_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x8000c, v0
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x80004, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> <i16 4, i16 8>)
ret <2 x i16> %result
}
@@ -3642,6 +3824,20 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_v2i16_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s2, s2, 1
+; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v1, s0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_v2i16_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s2, s0, 16
@@ -3655,7 +3851,6 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v1, s1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_v2i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -3669,21 +3864,6 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX10-NEXT: v_pk_lshlrev_b16 v1, v1, s0
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshr_v2i16_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s2, s2, 1
-; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, v0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v1, s0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -3728,6 +3908,24 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_v2i16_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s2, s2, 1
+; GFX11-NEXT: s_and_b32 s3, s1, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, s3, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_v2i16_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s3, s0, 16
@@ -3744,7 +3942,6 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX9-NEXT: v_pk_lshrrev_b16 v0, s2, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_v2i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s2, s0, 16
@@ -3761,25 +3958,6 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshr_v2i16_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s2, s2, 1
-; GFX11-NEXT: s_and_b32 s3, s1, 0xf000f
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, s3, v0
-; GFX11-NEXT: s_lshl_b32 s0, s0, s1
-; GFX11-NEXT: s_lshl_b32 s1, s2, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -3825,6 +4003,22 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_v2i16_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, s1, v0
+; GFX11-NEXT: s_lshr_b32 s1, s3, s2
+; GFX11-NEXT: s_lshr_b32 s0, s0, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_v2i16_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
@@ -3839,7 +4033,6 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_v2i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
@@ -3854,23 +4047,6 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshr_v2i16_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, s1, v0
-; GFX11-NEXT: s_lshr_b32 s1, s3, s2
-; GFX11-NEXT: s_lshr_b32 s0, s0, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -3950,158 +4126,177 @@ define amdgpu_ps i48 @s_fshr_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s6, s6, 1
+; GFX11-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s8
+; GFX11-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_lshr_b32 s8, s7, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX11-NEXT: s_lshl_b32 s4, s4, 1
+; GFX11-NEXT: s_lshr_b32 s6, s6, s7
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX11-NEXT: s_and_b32 s6, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s5, s7
+; GFX11-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s6, 16
+; GFX11-NEXT: s_lshr_b32 s5, s5, s6
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s6, s5, 15
-; GFX9-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX9-NEXT: s_lshr_b32 s6, s7, s6
-; GFX9-NEXT: s_andn2_b32 s7, 15, s5
-; GFX9-NEXT: s_lshl_b32 s8, s1, 1
-; GFX9-NEXT: s_lshl_b32 s7, s8, s7
-; GFX9-NEXT: s_or_b32 s6, s7, s6
-; GFX9-NEXT: s_lshr_b32 s7, s5, 16
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x100010
-; GFX9-NEXT: s_andn2_b32 s7, 15, s7
-; GFX9-NEXT: s_lshl_b32 s1, s1, 1
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT: s_lshl_b32 s7, s7, 1
+; GFX9-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s7
+; GFX9-NEXT: s_lshr_b32 s7, s0, 16
+; GFX9-NEXT: s_lshr_b32 s8, s4, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, s4
+; GFX9-NEXT: s_lshl_b32 s4, s7, s8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s7, s6, 16
+; GFX9-NEXT: s_lshr_b32 s4, s4, s6
+; GFX9-NEXT: s_lshr_b32 s2, s2, s7
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s5, 0xf000f
+; GFX9-NEXT: s_andn2_b32 s4, 0xf000f, s5
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX9-NEXT: s_lshl_b32 s5, s5, 1
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT: s_lshr_b32 s5, s1, 16
+; GFX9-NEXT: s_lshr_b32 s6, s4, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, s4
+; GFX9-NEXT: s_lshl_b32 s4, s5, s6
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_and_b32 s4, s3, 0xffff
; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX9-NEXT: s_lshl_b32 s1, s1, s7
+; GFX9-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-NEXT: s_lshr_b32 s2, s4, s2
; GFX9-NEXT: s_lshr_b32 s3, s3, s5
-; GFX9-NEXT: s_or_b32 s1, s1, s3
-; GFX9-NEXT: s_and_b32 s3, s4, 15
-; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX9-NEXT: s_lshr_b32 s3, s5, s3
-; GFX9-NEXT: s_andn2_b32 s5, 15, s4
-; GFX9-NEXT: s_lshl_b32 s6, s0, 1
-; GFX9-NEXT: s_lshl_b32 s5, s6, s5
-; GFX9-NEXT: s_or_b32 s3, s5, s3
-; GFX9-NEXT: s_lshr_b32 s5, s4, 16
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX9-NEXT: s_andn2_b32 s5, 15, s5
-; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX9-NEXT: s_lshl_b32 s0, s0, s5
-; GFX9-NEXT: s_lshr_b32 s2, s2, s4
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT: s_or_b32 s1, s1, s2
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
; GFX9-NEXT: s_or_b32 s0, s0, s2
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_v3i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s6, s5, 15
-; GFX10-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX10-NEXT: s_andn2_b32 s8, 15, s5
-; GFX10-NEXT: s_lshl_b32 s9, s1, 1
-; GFX10-NEXT: s_lshr_b32 s6, s7, s6
-; GFX10-NEXT: s_lshl_b32 s7, s9, s8
-; GFX10-NEXT: s_bfe_u32 s1, s1, 0x100010
-; GFX10-NEXT: s_or_b32 s6, s7, s6
-; GFX10-NEXT: s_lshr_b32 s7, s5, 16
-; GFX10-NEXT: s_lshl_b32 s1, s1, 1
-; GFX10-NEXT: s_andn2_b32 s7, 15, s7
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
-; GFX10-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX10-NEXT: s_lshl_b32 s1, s1, s7
-; GFX10-NEXT: s_lshr_b32 s3, s3, s5
-; GFX10-NEXT: s_and_b32 s5, s4, 15
-; GFX10-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX10-NEXT: s_andn2_b32 s8, 15, s4
-; GFX10-NEXT: s_lshl_b32 s9, s0, 1
-; GFX10-NEXT: s_lshr_b32 s5, s7, s5
-; GFX10-NEXT: s_lshl_b32 s7, s9, s8
+; GFX10-NEXT: s_lshr_b32 s6, s0, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT: s_lshl_b32 s6, s6, 1
+; GFX10-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX10-NEXT: s_andn2_b32 s4, 0xf000f, s4
+; GFX10-NEXT: s_lshr_b32 s6, s0, 16
; GFX10-NEXT: s_lshr_b32 s8, s4, 16
-; GFX10-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX10-NEXT: s_andn2_b32 s8, 15, s8
-; GFX10-NEXT: s_lshl_b32 s0, s0, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, s4
+; GFX10-NEXT: s_lshl_b32 s4, s6, s8
+; GFX10-NEXT: s_and_b32 s6, s2, 0xffff
; GFX10-NEXT: s_lshr_b32 s2, s2, 16
-; GFX10-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX10-NEXT: s_lshl_b32 s0, s0, s8
-; GFX10-NEXT: s_lshr_b32 s2, s2, s4
-; GFX10-NEXT: s_or_b32 s4, s7, s5
+; GFX10-NEXT: s_lshr_b32 s8, s7, 16
+; GFX10-NEXT: s_lshr_b32 s6, s6, s7
+; GFX10-NEXT: s_lshr_b32 s2, s2, s8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX10-NEXT: s_and_b32 s4, s5, 0xf000f
; GFX10-NEXT: s_or_b32 s0, s0, s2
-; GFX10-NEXT: s_or_b32 s1, s1, s3
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX10-NEXT: s_lshr_b32 s2, s1, 16
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX10-NEXT: s_lshl_b32 s2, s2, 1
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT: s_andn2_b32 s2, 0xf000f, s5
+; GFX10-NEXT: s_lshr_b32 s5, s1, 16
+; GFX10-NEXT: s_lshr_b32 s6, s2, 16
+; GFX10-NEXT: s_lshl_b32 s1, s1, s2
+; GFX10-NEXT: s_lshl_b32 s2, s5, s6
+; GFX10-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX10-NEXT: s_lshr_b32 s3, s3, 16
+; GFX10-NEXT: s_lshr_b32 s6, s4, 16
+; GFX10-NEXT: s_lshr_b32 s4, s5, s4
+; GFX10-NEXT: s_lshr_b32 s3, s3, s6
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s4, s3
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s3, s3, 16
+; GFX10-NEXT: s_or_b32 s1, s1, s2
+; GFX10-NEXT: s_or_b32 s0, s0, s3
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_fshr_v3i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s6, 1
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s4, 0xf000f
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s8
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s2, 0xffff
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s7, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x10001
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s6, s7
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s5
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s6, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s5, 0xf000f
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s5, s7
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s6, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s5, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
-; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: s_fshr_v3i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s6, s5, 15
-; GFX11-FAKE16-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s5
-; GFX11-FAKE16-NEXT: s_lshl_b32 s9, s1, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s7, s6
-; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s9, s8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 15
-; GFX11-FAKE16-NEXT: s_or_b32 s6, s7, s6
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0x1fffe
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s7, 15, s7
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-FAKE16-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s7
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s5
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s4, 15
-; GFX11-FAKE16-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s4
-; GFX11-FAKE16-NEXT: s_lshl_b32 s9, s0, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s7, s5
-; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s9, s8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-FAKE16-NEXT: s_lshl_b32 s6, s6, 1
+; GFX11-FAKE16-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 15
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s8, 15, s8
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x1fffe
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s6, s8
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s2, 0xffff
; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-FAKE16-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, s8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s4
-; GFX11-FAKE16-NEXT: s_or_b32 s4, s7, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s6, s7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s5, 0xf000f
; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s3
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s4, s0
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_and_not1_b32 s2, 0xf000f, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s5, s6
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s5, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, s6
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s4, s3
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
%cast = bitcast <3 x i16> %result to i48
@@ -4152,101 +4347,62 @@ define <3 x half> @v_fshr_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v4, v2
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v5, v3
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v7, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v7, 1
-; GFX9-NEXT: v_mov_b32_e32 v8, -1
-; GFX9-NEXT: v_lshlrev_b16_sdwa v7, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_xor_b32_sdwa v8, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b16_sdwa v6, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_lshlrev_b16_e32 v7, v8, v7
-; GFX9-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 1, v1
-; GFX9-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, v7, v1
-; GFX9-NEXT: v_lshrrev_b16_e32 v3, v5, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 1, v0
-; GFX9-NEXT: v_xor_b32_e32 v3, -1, v4
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, v3, v0
-; GFX9-NEXT: v_lshrrev_b16_e32 v2, v4, v2
+; GFX9-NEXT: v_and_b32_e32 v6, 0xf000f, v4
+; GFX9-NEXT: v_xor_b32_e32 v4, -1, v4
+; GFX9-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, v4, v0
+; GFX9-NEXT: v_pk_lshrrev_b16 v2, v6, v2
+; GFX9-NEXT: v_xor_b32_e32 v4, -1, v5
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v6, v0, s4
+; GFX9-NEXT: v_and_b32_e32 v2, 0xf000f, v5
+; GFX9-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, v4, v1
+; GFX9-NEXT: v_pk_lshrrev_b16 v2, v2, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v3i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
-; GFX10-NEXT: v_xor_b32_e32 v10, -1, v4
-; GFX10-NEXT: v_lshlrev_b16 v6, 1, v6
-; GFX10-NEXT: v_xor_b32_e32 v9, -1, v7
-; GFX10-NEXT: v_lshlrev_b16 v1, 1, v1
-; GFX10-NEXT: v_lshrrev_b16 v7, v7, v8
-; GFX10-NEXT: v_lshlrev_b16 v0, v10, v0
-; GFX10-NEXT: v_lshrrev_b16 v2, v4, v2
-; GFX10-NEXT: v_lshlrev_b16 v6, v9, v6
-; GFX10-NEXT: v_xor_b32_e32 v4, -1, v5
-; GFX10-NEXT: v_lshrrev_b16 v3, v5, v3
+; GFX10-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX10-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX10-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX10-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX10-NEXT: v_pk_lshrrev_b16 v2, v4, v2
+; GFX10-NEXT: v_pk_lshrrev_b16 v3, v5, v3
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, v6, v0
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, v7, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v5, v6, v7
-; GFX10-NEXT: v_lshlrev_b16 v1, v4, v1
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x5040100
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_v3i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, v6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, v7, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_fshr_v3i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 1, v0
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v10, -1, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 1, v6
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v9, -1, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 1, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v7, v7, v8
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, v10, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, v4, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, v9, v6
-; GFX11-FAKE16-NEXT: v_xor_b32_e32 v4, -1, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, v5, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, v6, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, v4, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x5040100
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
%cast.result = bitcast <3 x i16> %result to <3 x half>
ret <3 x half> %cast.result
@@ -4326,6 +4482,45 @@ define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s6, s6, 1
+; GFX11-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s8
+; GFX11-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_lshr_b32 s8, s7, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX11-NEXT: s_lshl_b32 s4, s4, 1
+; GFX11-NEXT: s_lshr_b32 s6, s6, s7
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX11-NEXT: s_and_b32 s6, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s5, s7
+; GFX11-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s6, 16
+; GFX11-NEXT: s_lshr_b32 s5, s5, s6
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s7, s0, 16
@@ -4365,7 +4560,6 @@ define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
; GFX9-NEXT: s_or_b32 s1, s1, s2
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s6, s0, 16
@@ -4405,46 +4599,6 @@ define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s6, s6, 1
-; GFX11-NEXT: s_and_b32 s7, s4, 0xf000f
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-NEXT: s_lshl_b32 s4, s6, s8
-; GFX11-NEXT: s_and_b32 s6, s2, 0xffff
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s4
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_lshr_b32 s8, s7, 16
-; GFX11-NEXT: s_lshl_b32 s1, s1, 0x10001
-; GFX11-NEXT: s_lshl_b32 s4, s4, 1
-; GFX11-NEXT: s_lshr_b32 s6, s6, s7
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
-; GFX11-NEXT: s_and_b32 s6, s5, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-NEXT: s_lshl_b32 s4, s5, s7
-; GFX11-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_lshr_b32 s7, s6, 16
-; GFX11-NEXT: s_lshr_b32 s5, s5, s6
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
%result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <2 x i32>
ret <2 x i32> %cast.result
@@ -4507,6 +4661,26 @@ define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX8-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v4, v2
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v5, v3
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v7, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4525,7 +4699,6 @@ define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX9-NEXT: v_pk_lshrrev_b16 v2, v2, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4544,51 +4717,30 @@ define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+ %result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
+ %cast.result = bitcast <4 x i16> %result to <4 x half>
+ ret <4 x half> %cast.result
+}
+
+define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt) {
+; GCN-LABEL: s_fshr_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GCN-NEXT: s_not_b32 s5, s4
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
+; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, v4, v2
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, v5, v3
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v7, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
- %result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
- %cast.result = bitcast <4 x i16> %result to <4 x half>
- ret <4 x half> %cast.result
-}
-
-define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt) {
-; GFX6-LABEL: s_fshr_i64:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX6-NEXT: s_not_b32 s5, s4
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshr_i64:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: s_not_b32 s5, s4
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: ; return to shader part epilog
-;
+; GFX11-LABEL: s_fshr_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s5, s4
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -4597,7 +4749,6 @@ define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -4606,16 +4757,6 @@ define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s5, s4
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -4709,6 +4850,19 @@ define i64 @v_fshr_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
+; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v5, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4720,7 +4874,6 @@ define i64 @v_fshr_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4732,20 +4885,6 @@ define i64 @v_fshr_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
-; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v5, v[2:3]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -4769,6 +4908,14 @@ define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i64_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v4, v0
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v1, v4, 27, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i64_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4776,7 +4923,6 @@ define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
; GFX9-NEXT: v_lshl_or_b32 v1, v4, 27, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i64_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4784,15 +4930,6 @@ define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
; GFX10-NEXT: v_lshl_or_b32 v1, v4, 27, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_i64_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v1, v4, 27, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 5)
ret i64 %result
}
@@ -4830,38 +4967,27 @@ define i64 @v_fshr_i64_48(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i64_48:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
+; GFX11-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX11-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i64_48:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i64_48:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: v_fshr_i64_48:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_fshr_i64_48:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 48)
ret i64 %result
}
@@ -4889,6 +5015,19 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX8-NEXT: v_or_b32_e32 v1, v2, v4
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i64_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_bfi_b32 v1, v0, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v2, 63, v0
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i64_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -4899,7 +5038,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX9-NEXT: v_or_b32_e32 v0, v1, v3
; GFX9-NEXT: v_or_b32_e32 v1, v2, v4
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i64_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_bfi_b32 v1, v0, 0, 63
@@ -4910,20 +5048,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshr_i64_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_bfi_b32 v1, v0, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v2, 63, v0
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -4956,6 +5080,18 @@ define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i64_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 63
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i64_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -4968,7 +5104,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i64_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -4981,19 +5116,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshr_i64_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s3, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 63
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -5024,6 +5146,18 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: v_fshr_i64_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i64_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
@@ -5035,7 +5169,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: v_fshr_i64_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
@@ -5047,53 +5180,39 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_fshr_i64_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
}
define amdgpu_ps <2 x i64> @s_fshr_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs, <2 x i64> inreg %amt) {
-; GFX6-LABEL: s_fshr_v2i64:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX6-NEXT: s_not_b32 s9, s8
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX6-NEXT: s_not_b32 s4, s10
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s4
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
-; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshr_v2i64:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: s_not_b32 s9, s8
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX8-NEXT: s_not_b32 s4, s10
-; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], s4
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
-; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshr_v2i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GCN-NEXT: s_not_b32 s9, s8
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
+; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GCN-NEXT: s_not_b32 s4, s10
+; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], s4
+; GCN-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
+; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GCN-NEXT: ; return to shader part epilog
;
+; GFX11-LABEL: s_fshr_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s9, s8
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
+; GFX11-NEXT: s_not_b32 s9, s10
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s10
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5107,7 +5226,6 @@ define amdgpu_ps <2 x i64> @s_fshr_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: ; return to shader part epilog
-;
; GFX10-LABEL: s_fshr_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5121,20 +5239,6 @@ define amdgpu_ps <2 x i64> @s_fshr_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: s_fshr_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s9, s8
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX11-NEXT: s_not_b32 s9, s10
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s10
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5178,6 +5282,28 @@ define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: v_bfi_b32 v9, v8, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v8, 63, v8
+; GFX11-NEXT: v_bfi_b32 v11, v10, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v10, 63, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5196,7 +5322,6 @@ define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5215,29 +5340,6 @@ define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX11-NEXT: v_bfi_b32 v9, v8, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v8, 63, v8
-; GFX11-NEXT: v_bfi_b32 v11, v10, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v10, 63, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5290,50 +5392,6 @@ define amdgpu_ps i128 @s_fshr_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshr_i128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s10, s8, 64
-; GFX9-NEXT: s_mov_b32 s11, 0
-; GFX9-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX9-NEXT: s_cselect_b32 s3, s1, s3
-; GFX9-NEXT: s_cselect_b32 s2, s0, s2
-; GFX9-NEXT: s_cselect_b32 s1, s7, s1
-; GFX9-NEXT: s_cselect_b32 s0, s6, s0
-; GFX9-NEXT: s_cselect_b32 s4, s4, s6
-; GFX9-NEXT: s_cselect_b32 s5, s5, s7
-; GFX9-NEXT: s_lshr_b64 s[6:7], s[0:1], s8
-; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX9-NEXT: s_not_b32 s9, s8
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshr_i128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s10, s8, 64
-; GFX10-NEXT: s_mov_b32 s11, 0
-; GFX10-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX10-NEXT: s_cselect_b32 s3, s1, s3
-; GFX10-NEXT: s_cselect_b32 s2, s0, s2
-; GFX10-NEXT: s_cselect_b32 s1, s7, s1
-; GFX10-NEXT: s_cselect_b32 s0, s6, s0
-; GFX10-NEXT: s_cselect_b32 s4, s4, s6
-; GFX10-NEXT: s_cselect_b32 s5, s5, s7
-; GFX10-NEXT: s_lshr_b64 s[6:7], s[0:1], s8
-; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX10-NEXT: s_not_b32 s9, s8
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
-; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: s_fshr_i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s10, s8, 64
@@ -5356,6 +5414,96 @@ define amdgpu_ps i128 @s_fshr_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshr_i128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b64 s[10:11], s[0:1], 1
+; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX9-NEXT: s_lshr_b32 s0, s1, 31
+; GFX9-NEXT: s_or_b32 s2, s2, s0
+; GFX9-NEXT: s_andn2_b32 s0, 0x7f, s8
+; GFX9-NEXT: s_not_b32 s9, s8
+; GFX9-NEXT: s_sub_i32 s16, s0, 64
+; GFX9-NEXT: s_sub_i32 s12, 64, s0
+; GFX9-NEXT: s_cmp_lt_u32 s0, 64
+; GFX9-NEXT: s_cselect_b32 s17, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s0, 0
+; GFX9-NEXT: s_cselect_b32 s18, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[12:13], s[10:11], s12
+; GFX9-NEXT: s_lshl_b64 s[14:15], s[2:3], s9
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[10:11], s9
+; GFX9-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX9-NEXT: s_lshl_b64 s[10:11], s[10:11], s16
+; GFX9-NEXT: s_cmp_lg_u32 s17, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX9-NEXT: s_cselect_b64 s[10:11], s[12:13], s[10:11]
+; GFX9-NEXT: s_cmp_lg_u32 s18, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX9-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX9-NEXT: s_sub_i32 s14, s9, 64
+; GFX9-NEXT: s_sub_i32 s12, 64, s9
+; GFX9-NEXT: s_cmp_lt_u32 s9, 64
+; GFX9-NEXT: s_cselect_b32 s15, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s9, 0
+; GFX9-NEXT: s_cselect_b32 s16, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[6:7], s8
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[4:5], s8
+; GFX9-NEXT: s_lshl_b64 s[12:13], s[6:7], s12
+; GFX9-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
+; GFX9-NEXT: s_cmp_lg_u32 s15, 0
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX9-NEXT: s_cmp_lg_u32 s16, 0
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-NEXT: s_cmp_lg_u32 s15, 0
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[10:11], 0
+; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshr_i128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_lshr_b32 s9, s1, 31
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT: s_or_b32 s2, s2, s9
+; GFX10-NEXT: s_andn2_b32 s9, 0x7f, s8
+; GFX10-NEXT: s_not_b32 s14, s8
+; GFX10-NEXT: s_sub_i32 s16, s9, 64
+; GFX10-NEXT: s_sub_i32 s10, 64, s9
+; GFX10-NEXT: s_cmp_lt_u32 s9, 64
+; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[10:11], s[0:1], s10
+; GFX10-NEXT: s_lshl_b64 s[12:13], s[2:3], s14
+; GFX10-NEXT: s_lshl_b64 s[14:15], s[0:1], s14
+; GFX10-NEXT: s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s16
+; GFX10-NEXT: s_cmp_lg_u32 s17, 0
+; GFX10-NEXT: s_cselect_b64 s[12:13], s[14:15], 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT: s_and_b32 s0, s8, 0x7f
+; GFX10-NEXT: s_sub_i32 s14, s0, 64
+; GFX10-NEXT: s_sub_i32 s9, 64, s0
+; GFX10-NEXT: s_cmp_lt_u32 s0, 64
+; GFX10-NEXT: s_cselect_b32 s15, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s0, 0
+; GFX10-NEXT: s_cselect_b32 s16, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], s8
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], s9
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[6:7], s8
+; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
+; GFX10-NEXT: s_cmp_lg_u32 s15, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX10-NEXT: s_cmp_lg_u32 s16, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s15, 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[8:9], 0
+; GFX10-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
+; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -5413,57 +5561,6 @@ define i128 @v_fshr_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_fshr_i128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX9-NEXT: v_mov_b32_e32 v10, 0
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[9:10]
-; GFX9-NEXT: v_not_b32_e32 v13, v8
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v7, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v6, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[9:10], v8, v[4:5]
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], 1, v[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX9-NEXT: v_lshlrev_b64 v[11:12], v13, v[4:5]
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], v8, v[6:7]
-; GFX9-NEXT: v_lshlrev_b64 v[5:6], v13, v[0:1]
-; GFX9-NEXT: v_or_b32_e32 v4, v12, v10
-; GFX9-NEXT: v_or_b32_e32 v0, v11, v9
-; GFX9-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX9-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_fshr_i128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX10-NEXT: v_mov_b32_e32 v10, 0
-; GFX10-NEXT: v_not_b32_e32 v11, v8
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
-; GFX10-NEXT: v_cndmask_b32_e32 v10, v1, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v0, v6, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], 1, v[9:10]
-; GFX10-NEXT: v_lshrrev_b64 v[6:7], v8, v[9:10]
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX10-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
-; GFX10-NEXT: v_lshlrev_b64 v[8:9], v11, v[0:1]
-; GFX10-NEXT: v_or_b32_e32 v0, v2, v4
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v5
-; GFX10-NEXT: v_or_b32_e32 v2, v8, v6
-; GFX10-NEXT: v_or_b32_e32 v3, v9, v7
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_fshr_i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5489,6 +5586,99 @@ define i128 @v_fshr_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX11-NEXT: v_or_b32_e32 v2, v8, v6
; GFX11-NEXT: v_or_b32_e32 v3, v9, v7
; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: v_fshr_i128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX9-NEXT: v_lshlrev_b64 v[9:10], 1, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0x7f
+; GFX9-NEXT: v_bfi_b32 v15, v8, 0, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 64, v15
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, v[9:10]
+; GFX9-NEXT: v_lshlrev_b64 v[11:12], v15, v[2:3]
+; GFX9-NEXT: v_add_u32_e32 v16, 0xffffffc0, v15
+; GFX9-NEXT: v_lshlrev_b64 v[13:14], v15, v[9:10]
+; GFX9-NEXT: v_or_b32_e32 v11, v0, v11
+; GFX9-NEXT: v_or_b32_e32 v12, v1, v12
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v16, v[9:10]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v13, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v13, 0, v14, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v15
+; GFX9-NEXT: v_and_b32_e32 v14, 0x7f, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v0, v2, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 64, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v1, v3, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v14, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], v2, v[6:7]
+; GFX9-NEXT: v_add_u32_e32 v15, 0xffffffc0, v14
+; GFX9-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v15, v[6:7]
+; GFX9-NEXT: v_lshrrev_b64 v[8:9], v14, v[6:7]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX9-NEXT: v_or_b32_e32 v1, v13, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v11, v2
+; GFX9-NEXT: v_or_b32_e32 v3, v12, v3
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_fshr_i128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX10-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
+; GFX10-NEXT: v_lshrrev_b32_e32 v9, 31, v1
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX10-NEXT: v_and_b32_e32 v19, 0x7f, v8
+; GFX10-NEXT: v_sub_nc_u32_e32 v10, 64, v18
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v9
+; GFX10-NEXT: v_add_nc_u32_e32 v14, 0xffffffc0, v18
+; GFX10-NEXT: v_sub_nc_u32_e32 v16, 64, v19
+; GFX10-NEXT: v_lshlrev_b64 v[12:13], v18, v[0:1]
+; GFX10-NEXT: v_lshrrev_b64 v[8:9], v10, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[10:11], v18, v[2:3]
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v18
+; GFX10-NEXT: v_lshrrev_b64 v[14:15], v19, v[4:5]
+; GFX10-NEXT: v_lshlrev_b64 v[16:17], v16, v[6:7]
+; GFX10-NEXT: v_cmp_gt_u32_e64 s5, 64, v19
+; GFX10-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 0xffffffc0, v19
+; GFX10-NEXT: v_or_b32_e32 v11, v9, v11
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v18
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v19
+; GFX10-NEXT: v_cndmask_b32_e32 v20, v0, v8, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[8:9], v10, v[6:7]
+; GFX10-NEXT: v_or_b32_e32 v0, v14, v16
+; GFX10-NEXT: v_or_b32_e32 v10, v15, v17
+; GFX10-NEXT: v_cndmask_b32_e32 v11, v1, v11, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v20, v2, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v0, s5
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v19, v[6:7]
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v9, v10, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v13, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, v4, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v6, v5, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, v0, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, v1, s5
+; GFX10-NEXT: v_or_b32_e32 v0, v12, v4
+; GFX10-NEXT: v_or_b32_e32 v1, v7, v5
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX10-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -5558,68 +5748,6 @@ define amdgpu_ps <4 x float> @v_fshr_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX8-NEXT: v_or_b32_e32 v2, v7, v4
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: v_fshr_i128_ssv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[1:2]
-; GFX9-NEXT: v_mov_b32_e32 v4, s7
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v8, s1
-; GFX9-NEXT: v_mov_b32_e32 v9, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v8, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[4:5]
-; GFX9-NEXT: v_not_b32_e32 v10, v0
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], v0, v[1:2]
-; GFX9-NEXT: v_lshlrev_b64 v[6:7], v10, v[6:7]
-; GFX9-NEXT: v_lshrrev_b64 v[4:5], v0, v[4:5]
-; GFX9-NEXT: v_or_b32_e32 v1, v7, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v3, v8, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[7:8], 1, v[7:8]
-; GFX9-NEXT: v_or_b32_e32 v0, v6, v2
-; GFX9-NEXT: v_lshlrev_b64 v[7:8], v10, v[7:8]
-; GFX9-NEXT: v_or_b32_e32 v3, v8, v5
-; GFX9-NEXT: v_or_b32_e32 v2, v7, v4
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: v_fshr_i128_ssv:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: v_mov_b32_e32 v5, s1
-; GFX10-NEXT: v_mov_b32_e32 v7, s0
-; GFX10-NEXT: v_mov_b32_e32 v8, s4
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[1:2]
-; GFX10-NEXT: v_mov_b32_e32 v1, s7
-; GFX10-NEXT: v_not_b32_e32 v9, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s7, v3, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v3, s6
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s1, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s3, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s2, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s0, v3, vcc_lo
-; GFX10-NEXT: v_lshlrev_b64 v[5:6], 1, v[5:6]
-; GFX10-NEXT: v_lshrrev_b64 v[1:2], v0, v[1:2]
-; GFX10-NEXT: v_lshlrev_b64 v[7:8], 1, v[3:4]
-; GFX10-NEXT: v_lshrrev_b64 v[3:4], v0, v[3:4]
-; GFX10-NEXT: v_lshlrev_b64 v[5:6], v9, v[5:6]
-; GFX10-NEXT: v_lshlrev_b64 v[7:8], v9, v[7:8]
-; GFX10-NEXT: v_or_b32_e32 v0, v7, v1
-; GFX10-NEXT: v_or_b32_e32 v1, v8, v2
-; GFX10-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX10-NEXT: v_or_b32_e32 v3, v6, v4
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: v_fshr_i128_ssv:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 64, v0
@@ -5652,6 +5780,101 @@ define amdgpu_ps <4 x float> @v_fshr_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX11-NEXT: v_or_b32_e32 v2, v5, v3
; GFX11-NEXT: v_or_b32_e32 v3, v6, v4
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: v_fshr_i128_ssv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[0:1], 1
+; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX9-NEXT: s_lshr_b32 s0, s1, 31
+; GFX9-NEXT: v_bfi_b32 v7, v0, 0, v1
+; GFX9-NEXT: s_or_b32 s2, s2, s0
+; GFX9-NEXT: v_sub_u32_e32 v1, 64, v7
+; GFX9-NEXT: v_lshrrev_b64 v[1:2], v1, s[8:9]
+; GFX9-NEXT: v_lshlrev_b64 v[3:4], v7, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v8, 0xffffffc0, v7
+; GFX9-NEXT: v_lshlrev_b64 v[5:6], v7, s[8:9]
+; GFX9-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX9-NEXT: v_lshlrev_b64 v[1:2], v8, s[8:9]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v8, 0, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX9-NEXT: v_and_b32_e32 v10, 0x7f, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v2, v4, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 64, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v10, s[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], v2, s[6:7]
+; GFX9-NEXT: v_add_u32_e32 v11, 0xffffffc0, v10
+; GFX9-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v11, s[6:7]
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], v10, s[6:7]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX9-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX9-NEXT: v_or_b32_e32 v3, v9, v3
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: v_fshr_i128_ssv:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_bfi_b32 v11, v0, 0, 0x7f
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], 1
+; GFX10-NEXT: s_lshr_b32 s2, s1, 31
+; GFX10-NEXT: v_and_b32_e32 v12, 0x7f, v0
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, 64, v11
+; GFX10-NEXT: s_or_b32 s8, s8, s2
+; GFX10-NEXT: v_add_nc_u32_e32 v0, 0xffffffc0, v11
+; GFX10-NEXT: v_lshlrev_b64 v[3:4], v11, s[8:9]
+; GFX10-NEXT: v_sub_nc_u32_e32 v9, 64, v12
+; GFX10-NEXT: v_lshrrev_b64 v[1:2], v1, s[0:1]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v13, 0xffffffc0, v12
+; GFX10-NEXT: v_lshrrev_b64 v[7:8], v12, s[4:5]
+; GFX10-NEXT: v_lshlrev_b64 v[9:10], v9, s[6:7]
+; GFX10-NEXT: v_lshlrev_b64 v[5:6], v11, s[0:1]
+; GFX10-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v0, s[0:1]
+; GFX10-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX10-NEXT: v_cmp_gt_u32_e64 s1, 64, v12
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v11
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v12
+; GFX10-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v14, v0, v3, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], v13, s[6:7]
+; GFX10-NEXT: v_or_b32_e32 v0, v7, v9
+; GFX10-NEXT: v_or_b32_e32 v7, v8, v10
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v0, s1
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v12, s[6:7]
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v14, s8, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, s9, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s4, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s5, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, v0, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, v1, s1
+; GFX10-NEXT: v_or_b32_e32 v0, v5, v2
+; GFX10-NEXT: v_or_b32_e32 v1, v6, v3
+; GFX10-NEXT: v_or_b32_e32 v2, v7, v8
+; GFX10-NEXT: v_or_b32_e32 v3, v4, v9
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -5713,60 +5936,6 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX8-NEXT: v_or_b32_e32 v2, s0, v2
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: v_fshr_i128_svs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_not_b32 s8, s4
-; GFX9-NEXT: s_and_b32 s6, s4, 64
-; GFX9-NEXT: s_mov_b32 s7, 0
-; GFX9-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], 1, v[2:3]
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], s4, v[2:3]
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s1, s1, s3
-; GFX9-NEXT: s_cselect_b32 s0, s0, s2
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], s8, v[4:5]
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX9-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX9-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX9-NEXT: v_or_b32_e32 v2, s0, v2
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: v_fshr_i128_svs:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s6, s4, 64
-; GFX10-NEXT: s_mov_b32 s7, 0
-; GFX10-NEXT: s_not_b32 s5, s4
-; GFX10-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s1, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s0, v2, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT: v_lshlrev_b64 v[5:6], 1, v[3:4]
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX10-NEXT: v_lshrrev_b64 v[2:3], s4, v[3:4]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, exec_lo
-; GFX10-NEXT: s_cselect_b32 s1, s1, s3
-; GFX10-NEXT: s_cselect_b32 s0, s0, s2
-; GFX10-NEXT: v_lshlrev_b64 v[5:6], s5, v[5:6]
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
-; GFX10-NEXT: v_or_b32_e32 v2, s0, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v6, v1
-; GFX10-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: v_fshr_i128_svs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s6, s4, 64
@@ -5793,6 +5962,122 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX11-NEXT: v_or_b32_e32 v1, v6, v1
; GFX11-NEXT: v_or_b32_e32 v3, s1, v3
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: v_fshr_i128_svs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b64 s[6:7], s[0:1], 1
+; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX9-NEXT: s_lshr_b32 s0, s1, 31
+; GFX9-NEXT: s_or_b32 s2, s2, s0
+; GFX9-NEXT: s_andn2_b32 s0, 0x7f, s4
+; GFX9-NEXT: s_not_b32 s5, s4
+; GFX9-NEXT: s_sub_i32 s12, s0, 64
+; GFX9-NEXT: s_sub_i32 s8, 64, s0
+; GFX9-NEXT: s_cmp_lt_u32 s0, 64
+; GFX9-NEXT: s_cselect_b32 s13, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s0, 0
+; GFX9-NEXT: s_cselect_b32 s14, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[6:7], s8
+; GFX9-NEXT: s_lshl_b64 s[10:11], s[2:3], s5
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[6:7], s5
+; GFX9-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], s12
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX9-NEXT: s_cmp_lg_u32 s14, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT: s_and_b32 s4, s4, 0x7f
+; GFX9-NEXT: s_sub_i32 s5, s4, 64
+; GFX9-NEXT: s_sub_i32 s6, 64, s4
+; GFX9-NEXT: s_cmp_lt_u32 s4, 64
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], s4, v[0:1]
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], s6, v[2:3]
+; GFX9-NEXT: s_cselect_b32 s8, 1, 0
+; GFX9-NEXT: v_lshrrev_b64 v[8:9], s4, v[2:3]
+; GFX9-NEXT: v_lshrrev_b64 v[2:3], s5, v[2:3]
+; GFX9-NEXT: s_cmp_lg_u32 s7, 0
+; GFX9-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_cmp_lg_u32 s8, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: v_fshr_i128_svs:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_lshr_b32 s5, s1, 31
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT: s_or_b32 s2, s2, s5
+; GFX10-NEXT: s_andn2_b32 s5, 0x7f, s4
+; GFX10-NEXT: s_not_b32 s10, s4
+; GFX10-NEXT: s_sub_i32 s12, s5, 64
+; GFX10-NEXT: s_sub_i32 s6, 64, s5
+; GFX10-NEXT: s_cmp_lt_u32 s5, 64
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], s10
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[0:1], s10
+; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT: s_and_b32 s2, s4, 0x7f
+; GFX10-NEXT: s_sub_i32 s4, 64, s2
+; GFX10-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX10-NEXT: s_sub_i32 s3, s2, 64
+; GFX10-NEXT: s_cmp_lt_u32 s2, 64
+; GFX10-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX10-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s2, 0
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -5856,61 +6141,6 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: v_fshr_i128_vss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s6, s4, 64
-; GFX9-NEXT: s_mov_b32 s7, 0
-; GFX9-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, s3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
-; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX9-NEXT: v_lshlrev_b64 v[7:8], 1, v[5:6]
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT: s_cselect_b32 s1, s1, s3
-; GFX9-NEXT: s_cselect_b32 s0, s0, s2
-; GFX9-NEXT: s_not_b32 s2, s4
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX9-NEXT: v_lshlrev_b64 v[7:8], s2, v[7:8]
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
-; GFX9-NEXT: v_lshrrev_b64 v[2:3], s4, v[5:6]
-; GFX9-NEXT: v_lshlrev_b64 v[5:6], s2, v[0:1]
-; GFX9-NEXT: v_or_b32_e32 v4, s1, v8
-; GFX9-NEXT: v_or_b32_e32 v0, s0, v7
-; GFX9-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX9-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: v_fshr_i128_vss:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s6, s4, 64
-; GFX10-NEXT: s_mov_b32 s7, 0
-; GFX10-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v1, s3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v0, s2, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT: s_and_b32 s5, vcc_lo, exec_lo
-; GFX10-NEXT: s_cselect_b32 s1, s1, s3
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], 1, v[4:5]
-; GFX10-NEXT: s_cselect_b32 s0, s0, s2
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX10-NEXT: s_not_b32 s2, s4
-; GFX10-NEXT: v_lshrrev_b64 v[4:5], s4, v[4:5]
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], s2, v[2:3]
-; GFX10-NEXT: v_lshlrev_b64 v[6:7], s2, v[0:1]
-; GFX10-NEXT: v_or_b32_e32 v0, s0, v2
-; GFX10-NEXT: v_or_b32_e32 v1, s1, v3
-; GFX10-NEXT: v_or_b32_e32 v2, v6, v4
-; GFX10-NEXT: v_or_b32_e32 v3, v7, v5
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: v_fshr_i128_vss:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s6, s4, 64
@@ -5940,67 +6170,139 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v5
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: v_fshr_i128_vss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX9-NEXT: s_andn2_b32 s5, 0x7f, s4
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], 1, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX9-NEXT: s_sub_i32 s6, s5, 64
+; GFX9-NEXT: s_sub_i32 s7, 64, s5
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX9-NEXT: s_cmp_lt_u32 s5, 64
+; GFX9-NEXT: s_cselect_b32 s8, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s5, 0
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], s7, v[4:5]
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX9-NEXT: s_cselect_b32 s9, 1, 0
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], s5, v[4:5]
+; GFX9-NEXT: s_cmp_lg_u32 s8, 0
+; GFX9-NEXT: v_or_b32_e32 v6, v0, v6
+; GFX9-NEXT: v_or_b32_e32 v7, v1, v7
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], s6, v[4:5]
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_cmp_lg_u32 s9, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX9-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX9-NEXT: s_sub_i32 s10, s5, 64
+; GFX9-NEXT: s_sub_i32 s8, 64, s5
+; GFX9-NEXT: s_cmp_lt_u32 s5, 64
+; GFX9-NEXT: s_cselect_b32 s11, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s5, 0
+; GFX9-NEXT: s_cselect_b32 s12, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[2:3], s4
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[0:1], s4
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX9-NEXT: s_cmp_lg_u32 s12, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[6:7], 0
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX9-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX9-NEXT: v_or_b32_e32 v3, v7, v3
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: v_fshr_i128_vss:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 31, v1
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX10-NEXT: s_andn2_b32 s5, 0x7f, s4
+; GFX10-NEXT: s_sub_i32 s6, 64, s5
+; GFX10-NEXT: s_sub_i32 s7, s5, 64
+; GFX10-NEXT: s_cmp_lt_u32 s5, 64
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX10-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX10-NEXT: s_cselect_b32 s6, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
+; GFX10-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
+; GFX10-NEXT: s_cselect_b32 s8, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX10-NEXT: s_cselect_b32 s6, exec_lo, 0
+; GFX10-NEXT: s_and_b32 s7, s4, 0x7f
+; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: s_sub_i32 s12, s7, 64
+; GFX10-NEXT: s_sub_i32 s10, 64, s7
+; GFX10-NEXT: s_cmp_lt_u32 s7, 64
+; GFX10-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s7, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
+; GFX10-NEXT: s_cselect_b32 s7, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[0:1], s4
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[2:3], s10
+; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[2:3], s4
+; GFX10-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX10-NEXT: s_cmp_lg_u32 s7, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
+; GFX10-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX10-NEXT: v_or_b32_e32 v1, v7, v1
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
}
define amdgpu_ps i128 @s_fshr_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
-; GFX6-LABEL: s_fshr_i128_65:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_mov_b32 s4, s2
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[0:1], 1
-; GFX6-NEXT: s_lshl_b32 s9, s4, 31
-; GFX6-NEXT: s_mov_b32 s8, 0
-; GFX6-NEXT: s_or_b64 s[4:5], s[8:9], s[2:3]
-; GFX6-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX6-NEXT: s_lshl_b32 s9, s0, 31
-; GFX6-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
-; GFX6-NEXT: s_mov_b32 s0, s6
-; GFX6-NEXT: s_mov_b32 s3, s5
-; GFX6-NEXT: ; return to shader part epilog
-;
-; GFX8-LABEL: s_fshr_i128_65:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_mov_b32 s4, s2
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[0:1], 1
-; GFX8-NEXT: s_lshl_b32 s9, s4, 31
-; GFX8-NEXT: s_mov_b32 s8, 0
-; GFX8-NEXT: s_or_b64 s[4:5], s[8:9], s[2:3]
-; GFX8-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX8-NEXT: s_lshl_b32 s9, s0, 31
-; GFX8-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
-; GFX8-NEXT: s_mov_b32 s0, s6
-; GFX8-NEXT: s_mov_b32 s3, s5
-; GFX8-NEXT: ; return to shader part epilog
-;
-; GFX9-LABEL: s_fshr_i128_65:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[0:1], 1
-; GFX9-NEXT: s_lshl_b32 s9, s4, 31
-; GFX9-NEXT: s_mov_b32 s8, 0
-; GFX9-NEXT: s_or_b64 s[4:5], s[8:9], s[2:3]
-; GFX9-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX9-NEXT: s_lshl_b32 s9, s0, 31
-; GFX9-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
-; GFX9-NEXT: s_mov_b32 s0, s6
-; GFX9-NEXT: s_mov_b32 s3, s5
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshr_i128_65:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_mov_b32 s4, s2
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[0:1], 1
-; GFX10-NEXT: s_lshl_b32 s9, s4, 31
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX10-NEXT: s_or_b64 s[4:5], s[8:9], s[2:3]
-; GFX10-NEXT: s_lshl_b32 s9, s0, 31
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
-; GFX10-NEXT: s_mov_b32 s0, s6
-; GFX10-NEXT: ; return to shader part epilog
+; GCN-LABEL: s_fshr_i128_65:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s4, s2
+; GCN-NEXT: s_lshr_b64 s[2:3], s[0:1], 1
+; GCN-NEXT: s_lshl_b32 s9, s4, 31
+; GCN-NEXT: s_mov_b32 s8, 0
+; GCN-NEXT: s_or_b64 s[4:5], s[8:9], s[2:3]
+; GCN-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GCN-NEXT: s_lshl_b32 s9, s0, 31
+; GCN-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
+; GCN-NEXT: s_mov_b32 s0, s6
+; GCN-NEXT: s_mov_b32 s3, s5
+; GCN-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i128_65:
; GFX11: ; %bb.0:
@@ -6015,6 +6317,28 @@ define amdgpu_ps i128 @s_fshr_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
; GFX11-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
; GFX11-NEXT: s_mov_b32 s0, s6
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshr_i128_65:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_mov_b32 s4, 0
+; GFX9-NEXT: s_lshl_b32 s5, s0, 31
+; GFX9-NEXT: s_lshl_b32 s3, s2, 31
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[6:7], 1
+; GFX9-NEXT: s_or_b64 s[0:1], s[4:5], s[0:1]
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshr_i128_65:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_lshl_b32 s5, s0, 31
+; GFX10-NEXT: s_lshl_b32 s3, s2, 31
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[0:1], 1
+; GFX10-NEXT: s_or_b64 s[0:1], s[4:5], s[6:7]
+; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX10-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -6042,6 +6366,18 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX11-LABEL: v_fshr_i128_65:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v8, v2
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[6:7]
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshl_or_b32 v1, v0, 31, v5
+; GFX11-NEXT: v_lshl_or_b32 v3, v8, 31, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_mov_b32_e32 v0, v4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i128_65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6052,7 +6388,6 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshl_or_b32 v1, v0, 31, v5
; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX10-LABEL: v_fshr_i128_65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6063,19 +6398,6 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
; GFX10-NEXT: v_lshl_or_b32 v3, v8, 31, v3
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshr_i128_65:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[6:7]
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshl_or_b32 v1, v0, 31, v5
-; GFX11-NEXT: v_lshl_or_b32 v3, v8, 31, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_mov_b32_e32 v0, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -6165,84 +6487,6 @@ define amdgpu_ps <2 x i128> @s_fshr_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_fshr_v2i128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s18, s20, 64
-; GFX9-NEXT: s_mov_b32 s19, 0
-; GFX9-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX9-NEXT: s_cselect_b32 s7, s5, s7
-; GFX9-NEXT: s_cselect_b32 s6, s4, s6
-; GFX9-NEXT: s_cselect_b32 s5, s15, s5
-; GFX9-NEXT: s_cselect_b32 s4, s14, s4
-; GFX9-NEXT: s_cselect_b32 s12, s12, s14
-; GFX9-NEXT: s_cselect_b32 s13, s13, s15
-; GFX9-NEXT: s_lshr_b64 s[14:15], s[4:5], s20
-; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
-; GFX9-NEXT: s_not_b32 s17, s20
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 1
-; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], s17
-; GFX9-NEXT: s_lshr_b64 s[12:13], s[12:13], s20
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], s17
-; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], s[14:15]
-; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
-; GFX9-NEXT: s_and_b32 s18, s16, 64
-; GFX9-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX9-NEXT: s_cselect_b32 s3, s1, s3
-; GFX9-NEXT: s_cselect_b32 s2, s0, s2
-; GFX9-NEXT: s_cselect_b32 s1, s11, s1
-; GFX9-NEXT: s_cselect_b32 s0, s10, s0
-; GFX9-NEXT: s_cselect_b32 s8, s8, s10
-; GFX9-NEXT: s_cselect_b32 s9, s9, s11
-; GFX9-NEXT: s_lshr_b64 s[10:11], s[0:1], s16
-; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX9-NEXT: s_not_b32 s12, s16
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], s12
-; GFX9-NEXT: s_lshr_b64 s[8:9], s[8:9], s16
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
-; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
-; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_fshr_v2i128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s18, s20, 64
-; GFX10-NEXT: s_mov_b32 s19, 0
-; GFX10-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX10-NEXT: s_cselect_b32 s7, s5, s7
-; GFX10-NEXT: s_cselect_b32 s6, s4, s6
-; GFX10-NEXT: s_cselect_b32 s5, s15, s5
-; GFX10-NEXT: s_cselect_b32 s4, s14, s4
-; GFX10-NEXT: s_cselect_b32 s12, s12, s14
-; GFX10-NEXT: s_cselect_b32 s13, s13, s15
-; GFX10-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
-; GFX10-NEXT: s_not_b32 s17, s20
-; GFX10-NEXT: s_lshr_b64 s[14:15], s[4:5], s20
-; GFX10-NEXT: s_lshl_b64 s[4:5], s[4:5], 1
-; GFX10-NEXT: s_lshl_b64 s[6:7], s[6:7], s17
-; GFX10-NEXT: s_lshr_b64 s[12:13], s[12:13], s20
-; GFX10-NEXT: s_lshl_b64 s[4:5], s[4:5], s17
-; GFX10-NEXT: s_and_b32 s18, s16, 64
-; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[14:15]
-; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
-; GFX10-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX10-NEXT: s_cselect_b32 s3, s1, s3
-; GFX10-NEXT: s_cselect_b32 s2, s0, s2
-; GFX10-NEXT: s_cselect_b32 s1, s11, s1
-; GFX10-NEXT: s_cselect_b32 s0, s10, s0
-; GFX10-NEXT: s_cselect_b32 s8, s8, s10
-; GFX10-NEXT: s_cselect_b32 s9, s9, s11
-; GFX10-NEXT: s_lshr_b64 s[10:11], s[0:1], s16
-; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX10-NEXT: s_not_b32 s12, s16
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s16
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
-; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], s12
-; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
-; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX11-LABEL: s_fshr_v2i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s18, s20, 64
@@ -6282,6 +6526,180 @@ define amdgpu_ps <2 x i128> @s_fshr_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
; GFX11-NEXT: ; return to shader part epilog
+; GFX9-LABEL: s_fshr_v2i128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b64 s[18:19], s[0:1], 1
+; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX9-NEXT: s_lshr_b32 s0, s1, 31
+; GFX9-NEXT: s_or_b32 s2, s2, s0
+; GFX9-NEXT: s_andn2_b32 s0, 0x7f, s16
+; GFX9-NEXT: s_not_b32 s17, s16
+; GFX9-NEXT: s_sub_i32 s21, s0, 64
+; GFX9-NEXT: s_sub_i32 s22, 64, s0
+; GFX9-NEXT: s_cmp_lt_u32 s0, 64
+; GFX9-NEXT: s_cselect_b32 s26, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s0, 0
+; GFX9-NEXT: s_cselect_b32 s27, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[22:23], s[18:19], s22
+; GFX9-NEXT: s_lshl_b64 s[24:25], s[2:3], s17
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[18:19], s17
+; GFX9-NEXT: s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX9-NEXT: s_lshl_b64 s[18:19], s[18:19], s21
+; GFX9-NEXT: s_cmp_lg_u32 s26, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX9-NEXT: s_cselect_b64 s[18:19], s[22:23], s[18:19]
+; GFX9-NEXT: s_cmp_lg_u32 s27, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[18:19]
+; GFX9-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX9-NEXT: s_sub_i32 s21, s17, 64
+; GFX9-NEXT: s_sub_i32 s22, 64, s17
+; GFX9-NEXT: s_cmp_lt_u32 s17, 64
+; GFX9-NEXT: s_cselect_b32 s24, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s17, 0
+; GFX9-NEXT: s_cselect_b32 s25, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[18:19], s[10:11], s16
+; GFX9-NEXT: s_lshr_b64 s[16:17], s[8:9], s16
+; GFX9-NEXT: s_lshl_b64 s[22:23], s[10:11], s22
+; GFX9-NEXT: s_or_b64 s[16:17], s[16:17], s[22:23]
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[10:11], s21
+; GFX9-NEXT: s_cmp_lg_u32 s24, 0
+; GFX9-NEXT: s_cselect_b64 s[10:11], s[16:17], s[10:11]
+; GFX9-NEXT: s_cmp_lg_u32 s25, 0
+; GFX9-NEXT: s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT: s_cmp_lg_u32 s24, 0
+; GFX9-NEXT: s_cselect_b64 s[10:11], s[18:19], 0
+; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[4:5], 1
+; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
+; GFX9-NEXT: s_lshr_b32 s4, s5, 31
+; GFX9-NEXT: s_or_b32 s6, s6, s4
+; GFX9-NEXT: s_andn2_b32 s4, 0x7f, s20
+; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
+; GFX9-NEXT: s_not_b32 s16, s20
+; GFX9-NEXT: s_sub_i32 s18, s4, 64
+; GFX9-NEXT: s_sub_i32 s10, 64, s4
+; GFX9-NEXT: s_cmp_lt_u32 s4, 64
+; GFX9-NEXT: s_cselect_b32 s19, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s4, 0
+; GFX9-NEXT: s_cselect_b32 s21, 1, 0
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[8:9], s16
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[8:9], s10
+; GFX9-NEXT: s_lshl_b64 s[16:17], s[6:7], s16
+; GFX9-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX9-NEXT: s_lshl_b64 s[8:9], s[8:9], s18
+; GFX9-NEXT: s_cmp_lg_u32 s19, 0
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[4:5], 0
+; GFX9-NEXT: s_cselect_b64 s[8:9], s[10:11], s[8:9]
+; GFX9-NEXT: s_cmp_lg_u32 s21, 0
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT: s_and_b32 s8, s20, 0x7f
+; GFX9-NEXT: s_sub_i32 s18, s8, 64
+; GFX9-NEXT: s_sub_i32 s16, 64, s8
+; GFX9-NEXT: s_cmp_lt_u32 s8, 64
+; GFX9-NEXT: s_cselect_b32 s19, 1, 0
+; GFX9-NEXT: s_cmp_eq_u32 s8, 0
+; GFX9-NEXT: s_cselect_b32 s21, 1, 0
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[12:13], s20
+; GFX9-NEXT: s_lshl_b64 s[16:17], s[14:15], s16
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[14:15], s20
+; GFX9-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX9-NEXT: s_lshr_b64 s[14:15], s[14:15], s18
+; GFX9-NEXT: s_cmp_lg_u32 s19, 0
+; GFX9-NEXT: s_cselect_b64 s[10:11], s[10:11], s[14:15]
+; GFX9-NEXT: s_cmp_lg_u32 s21, 0
+; GFX9-NEXT: s_cselect_b64 s[10:11], s[12:13], s[10:11]
+; GFX9-NEXT: s_cmp_lg_u32 s19, 0
+; GFX9-NEXT: s_cselect_b64 s[8:9], s[8:9], 0
+; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_fshr_v2i128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_lshr_b32 s17, s1, 31
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT: s_or_b32 s2, s2, s17
+; GFX10-NEXT: s_andn2_b32 s17, 0x7f, s16
+; GFX10-NEXT: s_not_b32 s21, s16
+; GFX10-NEXT: s_sub_i32 s26, s17, 64
+; GFX10-NEXT: s_sub_i32 s18, 64, s17
+; GFX10-NEXT: s_cmp_lt_u32 s17, 64
+; GFX10-NEXT: s_cselect_b32 s27, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s17, 0
+; GFX10-NEXT: s_cselect_b32 s17, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[18:19], s[0:1], s18
+; GFX10-NEXT: s_lshl_b64 s[22:23], s[2:3], s21
+; GFX10-NEXT: s_lshl_b64 s[24:25], s[0:1], s21
+; GFX10-NEXT: s_or_b64 s[18:19], s[18:19], s[22:23]
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s26
+; GFX10-NEXT: s_cmp_lg_u32 s27, 0
+; GFX10-NEXT: s_cselect_b64 s[22:23], s[24:25], 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[18:19], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s17, 0
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT: s_and_b32 s0, s16, 0x7f
+; GFX10-NEXT: s_sub_i32 s21, s0, 64
+; GFX10-NEXT: s_sub_i32 s17, 64, s0
+; GFX10-NEXT: s_cmp_lt_u32 s0, 64
+; GFX10-NEXT: s_cselect_b32 s24, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s0, 0
+; GFX10-NEXT: s_cselect_b32 s25, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[8:9], s16
+; GFX10-NEXT: s_lshl_b64 s[18:19], s[10:11], s17
+; GFX10-NEXT: s_lshr_b64 s[16:17], s[10:11], s16
+; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[18:19]
+; GFX10-NEXT: s_lshr_b64 s[10:11], s[10:11], s21
+; GFX10-NEXT: s_cmp_lg_u32 s24, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT: s_cmp_lg_u32 s25, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT: s_cmp_lg_u32 s24, 0
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[16:17], 0
+; GFX10-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
+; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX10-NEXT: s_lshr_b32 s8, s5, 31
+; GFX10-NEXT: s_or_b64 s[0:1], s[22:23], s[0:1]
+; GFX10-NEXT: s_or_b32 s6, s6, s8
+; GFX10-NEXT: s_andn2_b32 s8, 0x7f, s20
+; GFX10-NEXT: s_lshl_b64 s[4:5], s[4:5], 1
+; GFX10-NEXT: s_not_b32 s16, s20
+; GFX10-NEXT: s_sub_i32 s18, s8, 64
+; GFX10-NEXT: s_sub_i32 s9, 64, s8
+; GFX10-NEXT: s_cmp_lt_u32 s8, 64
+; GFX10-NEXT: s_cselect_b32 s19, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s8, 0
+; GFX10-NEXT: s_cselect_b32 s21, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[4:5], s9
+; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], s16
+; GFX10-NEXT: s_lshl_b64 s[16:17], s[4:5], s16
+; GFX10-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT: s_lshl_b64 s[4:5], s[4:5], s18
+; GFX10-NEXT: s_cmp_lg_u32 s19, 0
+; GFX10-NEXT: s_cselect_b64 s[10:11], s[16:17], 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX10-NEXT: s_cmp_lg_u32 s21, 0
+; GFX10-NEXT: s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX10-NEXT: s_and_b32 s4, s20, 0x7f
+; GFX10-NEXT: s_sub_i32 s18, s4, 64
+; GFX10-NEXT: s_sub_i32 s8, 64, s4
+; GFX10-NEXT: s_cmp_lt_u32 s4, 64
+; GFX10-NEXT: s_cselect_b32 s19, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s4, 0
+; GFX10-NEXT: s_cselect_b32 s21, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[12:13], s20
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[14:15], s8
+; GFX10-NEXT: s_lshr_b64 s[16:17], s[14:15], s20
+; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[14:15], s18
+; GFX10-NEXT: s_cmp_lg_u32 s19, 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX10-NEXT: s_cmp_lg_u32 s21, 0
+; GFX10-NEXT: s_cselect_b64 s[4:5], s[12:13], s[4:5]
+; GFX10-NEXT: s_cmp_lg_u32 s19, 0
+; GFX10-NEXT: s_cselect_b64 s[8:9], s[16:17], 0
+; GFX10-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT: ; return to shader part epilog
%result = call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -6377,95 +6795,6 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX8-NEXT: v_mov_b32_e32 v5, v8
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_fshr_v2i128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX9-NEXT: v_mov_b32_e32 v18, 0
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[17:18]
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v11, v1, v11, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v10, v0, v10, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], 1, v[10:11]
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX9-NEXT: v_lshrrev_b64 v[8:9], v16, v[8:9]
-; GFX9-NEXT: v_lshrrev_b64 v[10:11], v16, v[10:11]
-; GFX9-NEXT: v_not_b32_e32 v16, v16
-; GFX9-NEXT: v_lshlrev_b64 v[2:3], v16, v[2:3]
-; GFX9-NEXT: v_lshlrev_b64 v[16:17], v16, v[0:1]
-; GFX9-NEXT: v_or_b32_e32 v1, v3, v9
-; GFX9-NEXT: v_or_b32_e32 v3, v17, v11
-; GFX9-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[17:18]
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v8
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v15, v13, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v5, v15, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v14, v4, v14, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[11:12], v20, v[8:9]
-; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[14:15]
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT: v_not_b32_e32 v13, v20
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
-; GFX9-NEXT: v_lshlrev_b64 v[17:18], v13, v[8:9]
-; GFX9-NEXT: v_or_b32_e32 v2, v16, v10
-; GFX9-NEXT: v_lshrrev_b64 v[6:7], v20, v[14:15]
-; GFX9-NEXT: v_lshlrev_b64 v[9:10], v13, v[4:5]
-; GFX9-NEXT: v_or_b32_e32 v8, v18, v12
-; GFX9-NEXT: v_or_b32_e32 v4, v17, v11
-; GFX9-NEXT: v_or_b32_e32 v7, v10, v7
-; GFX9-NEXT: v_or_b32_e32 v6, v9, v6
-; GFX9-NEXT: v_mov_b32_e32 v5, v8
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_fshr_v2i128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX10-NEXT: v_mov_b32_e32 v18, 0
-; GFX10-NEXT: v_not_b32_e32 v19, v16
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[17:18]
-; GFX10-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, 0, v[17:18]
-; GFX10-NEXT: v_cndmask_b32_e32 v22, v1, v11, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v21, v0, v10, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX10-NEXT: v_lshlrev_b64 v[10:11], 1, v[21:22]
-; GFX10-NEXT: v_cndmask_b32_e64 v18, v5, v15, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v17, v4, v14, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v7, v5, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v4, s4
-; GFX10-NEXT: v_lshrrev_b64 v[8:9], v16, v[8:9]
-; GFX10-NEXT: v_lshlrev_b64 v[10:11], v19, v[10:11]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX10-NEXT: v_lshlrev_b64 v[6:7], 1, v[17:18]
-; GFX10-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
-; GFX10-NEXT: v_cndmask_b32_e64 v13, v15, v13, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v12, v14, v12, s4
-; GFX10-NEXT: v_not_b32_e32 v0, v20
-; GFX10-NEXT: v_or_b32_e32 v1, v11, v9
-; GFX10-NEXT: v_lshrrev_b64 v[14:15], v16, v[21:22]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v19, v[2:3]
-; GFX10-NEXT: v_lshrrev_b64 v[11:12], v20, v[12:13]
-; GFX10-NEXT: v_lshlrev_b64 v[6:7], v0, v[6:7]
-; GFX10-NEXT: v_lshrrev_b64 v[16:17], v20, v[17:18]
-; GFX10-NEXT: v_lshlrev_b64 v[18:19], v0, v[4:5]
-; GFX10-NEXT: v_or_b32_e32 v0, v10, v8
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v15
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v14
-; GFX10-NEXT: v_or_b32_e32 v5, v7, v12
-; GFX10-NEXT: v_or_b32_e32 v4, v6, v11
-; GFX10-NEXT: v_or_b32_e32 v7, v19, v17
-; GFX10-NEXT: v_or_b32_e32 v6, v18, v16
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_fshr_v2i128:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6507,6 +6836,183 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX11-NEXT: v_or_b32_e32 v7, v19, v17
; GFX11-NEXT: v_or_b32_e32 v6, v18, v16
; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: v_fshr_v2i128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v19, 0x7f
+; GFX9-NEXT: v_lshlrev_b64 v[17:18], 1, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX9-NEXT: v_bfi_b32 v23, v16, 0, v19
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 64, v23
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, v[17:18]
+; GFX9-NEXT: v_lshlrev_b64 v[21:22], v23, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v26, 0x7f, v16
+; GFX9-NEXT: v_or_b32_e32 v24, v0, v21
+; GFX9-NEXT: v_sub_u32_e32 v0, 64, v26
+; GFX9-NEXT: v_or_b32_e32 v25, v1, v22
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, v[10:11]
+; GFX9-NEXT: v_lshrrev_b64 v[21:22], v26, v[8:9]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
+; GFX9-NEXT: v_or_b32_e32 v21, v21, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v23
+; GFX9-NEXT: v_or_b32_e32 v22, v22, v1
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, v[17:18]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v24, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v25, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v0, v2, s[4:5]
+; GFX9-NEXT: v_add_u32_e32 v0, 0xffffffc0, v26
+; GFX9-NEXT: v_lshlrev_b64 v[16:17], v23, v[17:18]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v1, v3, s[4:5]
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v0, v[10:11]
+; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v18, v0, v21, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v21, v1, v22, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], v26, v[10:11]
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v18, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v21, v9, vcc
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX9-NEXT: v_or_b32_e32 v0, v16, v8
+; GFX9-NEXT: v_or_b32_e32 v1, v17, v9
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[4:5]
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX9-NEXT: v_bfi_b32 v16, v20, 0, v19
+; GFX9-NEXT: v_or_b32_e32 v6, v6, v4
+; GFX9-NEXT: v_sub_u32_e32 v4, 64, v16
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX9-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], v4, v[8:9]
+; GFX9-NEXT: v_lshlrev_b64 v[10:11], v16, v[6:7]
+; GFX9-NEXT: v_add_u32_e32 v17, 0xffffffc0, v16
+; GFX9-NEXT: v_or_b32_e32 v10, v4, v10
+; GFX9-NEXT: v_or_b32_e32 v11, v5, v11
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v16, v[8:9]
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], v17, v[8:9]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-NEXT: v_and_b32_e32 v10, 0x7f, v20
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v6, vcc
+; GFX9-NEXT: v_sub_u32_e32 v6, 64, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v7, vcc
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], v10, v[12:13]
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], v6, v[14:15]
+; GFX9-NEXT: v_add_u32_e32 v11, 0xffffffc0, v10
+; GFX9-NEXT: v_or_b32_e32 v16, v4, v6
+; GFX9-NEXT: v_or_b32_e32 v19, v5, v7
+; GFX9-NEXT: v_lshrrev_b64 v[6:7], v11, v[14:15]
+; GFX9-NEXT: v_lshrrev_b64 v[4:5], v10, v[14:15]
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v16, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v19, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v13, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX9-NEXT: v_or_b32_e32 v4, v17, v6
+; GFX9-NEXT: v_or_b32_e32 v5, v18, v7
+; GFX9-NEXT: v_or_b32_e32 v6, v8, v10
+; GFX9-NEXT: v_or_b32_e32 v7, v9, v11
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_fshr_v2i128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX10-NEXT: v_bfi_b32 v25, v16, 0, 0x7f
+; GFX10-NEXT: v_lshrrev_b32_e32 v17, 31, v1
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX10-NEXT: v_and_b32_e32 v26, 0x7f, v16
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
+; GFX10-NEXT: v_sub_nc_u32_e32 v18, 64, v25
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v17
+; GFX10-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v25
+; GFX10-NEXT: v_lshlrev_b64 v[23:24], v25, v[0:1]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v25
+; GFX10-NEXT: v_lshrrev_b64 v[17:18], v18, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[21:22], v25, v[2:3]
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v19, v[0:1]
+; GFX10-NEXT: v_add_nc_u32_e32 v27, 0xffffffc0, v26
+; GFX10-NEXT: v_cndmask_b32_e32 v23, 0, v23, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v25
+; GFX10-NEXT: v_cndmask_b32_e32 v24, 0, v24, vcc_lo
+; GFX10-NEXT: v_or_b32_e32 v22, v18, v22
+; GFX10-NEXT: v_sub_nc_u32_e32 v18, 64, v26
+; GFX10-NEXT: v_or_b32_e32 v21, v17, v21
+; GFX10-NEXT: v_lshrrev_b64 v[16:17], v26, v[8:9]
+; GFX10-NEXT: v_bfi_b32 v25, v20, 0, 0x7f
+; GFX10-NEXT: v_cndmask_b32_e32 v22, v1, v22, vcc_lo
+; GFX10-NEXT: v_lshlrev_b64 v[18:19], v18, v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e32 v21, v0, v21, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v27, v[10:11]
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v26
+; GFX10-NEXT: v_cndmask_b32_e64 v22, v22, v3, s4
+; GFX10-NEXT: v_and_b32_e32 v20, 0x7f, v20
+; GFX10-NEXT: v_or_b32_e32 v16, v16, v18
+; GFX10-NEXT: v_cndmask_b32_e64 v21, v21, v2, s4
+; GFX10-NEXT: v_or_b32_e32 v17, v17, v19
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v26
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], v26, v[10:11]
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 31, v5
+; GFX10-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v25
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v8, s4
+; GFX10-NEXT: v_sub_nc_u32_e32 v8, 64, v25
+; GFX10-NEXT: v_or_b32_e32 v6, v6, v10
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v26, 0, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v27, 0, v3, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[2:3], v8, v[4:5]
+; GFX10-NEXT: v_lshlrev_b64 v[8:9], v25, v[6:7]
+; GFX10-NEXT: v_sub_nc_u32_e32 v18, 64, v20
+; GFX10-NEXT: v_lshlrev_b64 v[10:11], v25, v[4:5]
+; GFX10-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; GFX10-NEXT: v_or_b32_e32 v0, v23, v0
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v25
+; GFX10-NEXT: v_or_b32_e32 v8, v2, v8
+; GFX10-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v20
+; GFX10-NEXT: v_lshrrev_b64 v[16:17], v20, v[12:13]
+; GFX10-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
+; GFX10-NEXT: v_or_b32_e32 v2, v21, v26
+; GFX10-NEXT: v_or_b32_e32 v9, v3, v9
+; GFX10-NEXT: v_cndmask_b32_e32 v21, v4, v8, vcc_lo
+; GFX10-NEXT: v_lshrrev_b64 v[3:4], v23, v[14:15]
+; GFX10-NEXT: v_cmp_gt_u32_e64 s5, 64, v20
+; GFX10-NEXT: v_or_b32_e32 v8, v16, v18
+; GFX10-NEXT: v_or_b32_e32 v16, v17, v19
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v25
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v20
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v8, s5
+; GFX10-NEXT: v_lshrrev_b64 v[8:9], v20, v[14:15]
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v16, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v21, v6, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v5, v7, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v3, v12, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v12, v4, v13, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, v8, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, v9, s5
+; GFX10-NEXT: v_or_b32_e32 v1, v24, v1
+; GFX10-NEXT: v_or_b32_e32 v3, v22, v27
+; GFX10-NEXT: v_or_b32_e32 v4, v10, v5
+; GFX10-NEXT: v_or_b32_e32 v5, v11, v12
+; GFX10-NEXT: v_or_b32_e32 v6, v6, v8
+; GFX10-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX10-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -6543,3 +7049,5 @@ declare i128 @llvm.fshr.i128(i128, i128, i128) #0
declare <2 x i128> @llvm.fshr.v2i128(<2 x i128>, <2 x i128>, <2 x i128>) #0
attributes #0 = { nounwind readnone speculatable willreturn }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
index c55dff551f026..c26b2de6938fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
define amdgpu_ps half @fsub_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-FAKE16-LABEL: fsub_s16_uniform:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
index b06a203db76b5..ddf02e41e6ce8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=FIJI %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 < %s | FileCheck -check-prefix=FIJI %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GFX12 %s
define float @intrinsic_trunc_s(float inreg %val) {
; FIJI-LABEL: intrinsic_trunc_s:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
index 762d301a0f5c5..6fbf9a1612f57 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
@@ -1,5 +1,4 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# UNSUPPORTED: true
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
---
@@ -11,11 +10,11 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s64_to_s32
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = G_FPTRUNC [[COPY]](s64)
- ; CHECK-NEXT: $vgpr0 = COPY [[FPTRUNC]](s32)
- %0:_(s64) = COPY $vgpr0_vgpr1
- %1:_(s32) = G_FPTRUNC %0
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f32) = G_FPTRUNC [[COPY]](f64)
+ ; CHECK-NEXT: $vgpr0 = COPY [[FPTRUNC]](f32)
+ %0:_(f64) = COPY $vgpr0_vgpr1
+ %1:_(f32) = G_FPTRUNC %0
$vgpr0 = COPY %1
...
@@ -28,12 +27,12 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s32_to_s16
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY]](s32)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
+ ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[COPY]](f32)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](f16)
; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
- %0:_(s32) = COPY $vgpr0
- %1:_(s16) = G_FPTRUNC %0
+ %0:_(f32) = COPY $vgpr0
+ %1:_(f16) = G_FPTRUNC %0
%2:_(s32) = G_ANYEXT %1
$vgpr0 = COPY %2
...
@@ -47,14 +46,14 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s64_to_v2s32
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
- ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = G_FPTRUNC [[UV]](s64)
- ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s32) = G_FPTRUNC [[UV1]](s64)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTRUNC]](s32), [[FPTRUNC1]](s32)
- ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
- %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- %1:_(<2 x s32>) = G_FPTRUNC %0
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
+ ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f32) = G_FPTRUNC [[UV]](f64)
+ ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f32) = G_FPTRUNC [[UV1]](f64)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FPTRUNC]](f32), [[FPTRUNC1]](f32)
+ ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ %0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:_(<2 x f32>) = G_FPTRUNC %0
$vgpr0_vgpr1 = COPY %1
...
@@ -67,22 +66,22 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s32_to_v2s16
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
- ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV]](s32)
- ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV1]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[UV]](f32)
+ ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = G_FPTRUNC [[UV1]](f32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](f16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](f16)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x f16>)
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](s32), [[LSHR]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
- %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
- %1:_(<2 x s16>) = G_FPTRUNC %0
+ %0:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ %1:_(<2 x f16>) = G_FPTRUNC %0
%2:_(<2 x s32>) = G_ANYEXT %1
$vgpr0_vgpr1 = COPY %2
...
@@ -96,8 +95,8 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s64_to_s16
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](f64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2047
@@ -161,8 +160,8 @@ body: |
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C20]]
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SELECT3]]
; CHECK-NEXT: $vgpr0 = COPY [[OR7]](s32)
- %0:_(s64) = COPY $vgpr0_vgpr1
- %1:_(s16) = G_FPTRUNC %0
+ %0:_(f64) = COPY $vgpr0_vgpr1
+ %1:_(f16) = G_FPTRUNC %0
%2:_(s32) = G_ANYEXT %1
$vgpr0 = COPY %2
...
@@ -176,9 +175,9 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s64_to_v2s16
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
- ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
+ ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](f64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV3]], [[C]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2047
@@ -241,7 +240,7 @@ body: |
; CHECK-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 32768
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C20]]
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SELECT3]]
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](f64)
; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[UV5]], [[C]](s32)
; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C1]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[AND5]], [[C2]]
@@ -287,11 +286,11 @@ body: |
; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[OR7]], [[C21]]
; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[OR15]], [[C21]]
; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C19]](s32)
- ; CHECK-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[AND10]], [[SHL4]]
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR16]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
- %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- %1:_(<2 x s16>) = G_FPTRUNC %0
+ ; CHECK-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL4]]
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR16]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ %0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:_(<2 x f16>) = G_FPTRUNC %0
$vgpr0 = COPY %1
...
@@ -304,8 +303,8 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s64_to_s16_afn
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](f64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2047
@@ -369,8 +368,8 @@ body: |
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C20]]
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SELECT3]]
; CHECK-NEXT: $vgpr0 = COPY [[OR7]](s32)
- %0:_(s64) = COPY $vgpr0_vgpr1
- %1:_(s16) = G_FPTRUNC %0
+ %0:_(f64) = COPY $vgpr0_vgpr1
+ %1:_(f16) = G_FPTRUNC %0
%2:_(s32) = afn G_ANYEXT %1
$vgpr0 = COPY %2
...
@@ -384,20 +383,20 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s64_to_v2s16_afn
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
- ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV]](s64)
- ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = afn G_FPTRUNC [[FPTRUNC]](s32)
- ; CHECK-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV1]](s64)
- ; CHECK-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = afn G_FPTRUNC [[FPTRUNC2]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
+ ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV]](f64)
+ ; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = afn G_FPTRUNC [[FPTRUNC]](s32)
+ ; CHECK-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV1]](f64)
+ ; CHECK-NEXT: [[FPTRUNC3:%[0-9]+]]:_(f16) = afn G_FPTRUNC [[FPTRUNC2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](f16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](f16)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
- %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- %1:_(<2 x s16>) = afn G_FPTRUNC %0
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ %0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:_(<2 x f16>) = afn G_FPTRUNC %0
$vgpr0 = COPY %1
...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
index 8ca3c8dc7fe00..dc7e10ce8598a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
@@ -1,8 +1,28 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=PACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
+;xUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps half @image_load_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_f16
+ ; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(f16) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](f16)
+ ; PACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
; UNPACKED-LABEL: name: image_load_f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -23,8 +43,12 @@ define amdgpu_ps half @image_load_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](f16)
; UNPACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_f16
+ %tex = call half @llvm.amdgcn.image.load.2d.f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret half %tex
+}
+
+define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v2f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -40,15 +64,9 @@ define amdgpu_ps half @image_load_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(f16) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](f16)
- ; PACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
+ ; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- %tex = call half @llvm.amdgcn.image.load.2d.f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret half %tex
-}
-
-define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v2f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -76,8 +94,12 @@ define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_v2f16
+ %tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <2 x half> %tex
+}
+
+define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v3f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -93,14 +115,19 @@ define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32
; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- %tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <2 x half> %tex
-}
-
-define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x f16>)
+ ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x f16>)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; PACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; PACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
+ ; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; PACKED-NEXT: $vgpr0 = COPY [[UV]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
; UNPACKED-LABEL: name: image_load_v3f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -119,20 +146,27 @@ define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; UNPACKED-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[UV1]](i32)
- ; UNPACKED-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[UV2]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BUILD_VECTOR2]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[BUILD_VECTOR3]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_v3f16
+ %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <3 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v4f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -148,25 +182,11 @@ define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32
; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x f16>)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; PACKED-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x f16>)
- ; PACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; PACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; PACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV2]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<4 x f16>), addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x f16>)
+ ; PACKED-NEXT: $vgpr0 = COPY [[UV]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[UV1]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <3 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v4f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -200,8 +220,12 @@ define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_v4f16
+ %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <4 x half> %tex
+}
+
+define amdgpu_ps half @image_load_tfe_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -216,17 +240,13 @@ define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<4 x f16>), addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x f16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[UV1]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <4 x half> %tex
-}
-
-define amdgpu_ps half @image_load_tfe_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
+ ; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; PACKED-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
; UNPACKED-LABEL: name: image_load_tfe_f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -249,8 +269,15 @@ define amdgpu_ps half @image_load_tfe_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t)
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: $vgpr0 = COPY [[UV]](i32)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_tfe_f16
+ %res = call { half, i32 } @llvm.amdgcn.image.load.2d.sl_f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { half, i32 } %res, 0
+ %tfe = extractvalue { half, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret half %tex
+}
+
+define amdgpu_ps <2 x half> @image_load_tfe_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v2f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -267,19 +294,12 @@ define amdgpu_ps half @image_load_tfe_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t)
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
+ ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[UV]](i32)
; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- %res = call { half, i32 } @llvm.amdgcn.image.load.2d.sl_f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { half, i32 } %res, 0
- %tfe = extractvalue { half, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret half %tex
-}
-
-define amdgpu_ps <2 x half> @image_load_tfe_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v2f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -309,8 +329,15 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16(<8 x i32> inreg %rsrc, i32 %s,
; UNPACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_tfe_v2f16
+ %res = call { <2 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v2f16i32s.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <2 x half>, i32 } %res, 0
+ %tfe = extractvalue { <2 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <2 x half> %tex
+}
+
+define amdgpu_ps <3 x half> @image_load_tfe_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v3f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -327,20 +354,16 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16(<8 x i32> inreg %rsrc, i32 %s,
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[UV]](i32)
- ; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- %res = call { <2 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v2f16i32s.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <2 x half>, i32 } %res, 0
- %tfe = extractvalue { <2 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <2 x half> %tex
-}
-
-define amdgpu_ps <3 x half> @image_load_tfe_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
+ ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
+ ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV1]](i32)
+ ; PACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST1]](<2 x i16>)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST3]](<2 x f16>)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
; UNPACKED-LABEL: name: image_load_tfe_v3f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -360,21 +383,31 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16(<8 x i32> inreg %rsrc, i32 %s,
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; UNPACKED-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[UV1]](i32)
- ; UNPACKED-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[UV2]](i32)
; UNPACKED-NEXT: G_STORE [[UV3]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF1]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BUILD_VECTOR2]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[BUILD_VECTOR3]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v3f16
+ %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <3 x half>, i32 } %res, 0
+ %tfe = extractvalue { <3 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <3 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_tfe_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v4f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -391,34 +424,16 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16(<8 x i32> inreg %rsrc, i32 %s,
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<4 x f16>), addrspace 8)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV1]](i32)
- ; PACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x i16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x i16>), [[BITCAST1]](<2 x i16>)
- ; PACKED-NEXT: [[UV3:%[0-9]+]]:_(i16), [[UV4:%[0-9]+]]:_(i16), [[UV5:%[0-9]+]]:_(i16), [[UV6:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x i16>)
- ; PACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[UV3]](i16), [[UV4]](i16), [[UV5]](i16)
- ; PACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
- ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[UV7]](i16)
- ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[UV8]](i16)
- ; PACKED-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[UV9]](i16)
- ; PACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16), [[BITCAST4]](f16)
; PACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: [[UV10:%[0-9]+]]:_(f16), [[UV11:%[0-9]+]]:_(f16), [[UV12:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x f16>)
- ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<4 x f16>) = G_BUILD_VECTOR [[UV10]](f16), [[UV11]](f16), [[UV12]](f16), [[DEF1]](f16)
- ; PACKED-NEXT: [[UV13:%[0-9]+]]:_(<2 x f16>), [[UV14:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<4 x f16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV13]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[UV14]](<2 x f16>)
+ ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST1]](<2 x i16>)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST3]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <3 x half>, i32 } %res, 0
- %tfe = extractvalue { <3 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <3 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_tfe_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v4f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -454,34 +469,6 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16(<8 x i32> inreg %rsrc, i32 %s,
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v4f16
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<4 x f16>), addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
- ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV1]](i32)
- ; PACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
- ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST1]](<2 x i16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST3]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
%tex = extractvalue { <4 x half>, i32 } %res, 0
%tfe = extractvalue { <4 x half>, i32 } %res, 1
@@ -490,16 +477,6 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16(<8 x i32> inreg %rsrc, i32 %s,
}
define amdgpu_ps half @image_load_f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
- ; UNPACKED-LABEL: name: image_load_f16_dmask_0000
- ; UNPACKED: bb.1 (%ir-block.0):
- ; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
- ; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: $vgpr0 = COPY [[DEF]](i32)
- ; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; PACKED-LABEL: name: image_load_f16_dmask_0000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -509,11 +486,39 @@ define amdgpu_ps half @image_load_f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s,
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
; PACKED-NEXT: $vgpr0 = COPY [[DEF]](i32)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; UNPACKED-LABEL: name: image_load_f16_dmask_0000
+ ; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[DEF]](i32)
+ ; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%tex = call half @llvm.amdgcn.image.load.2d.f16.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret half %tex
}
define amdgpu_ps <2 x half> @image_load_v2f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v2f16_dmask_1000
+ ; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
; UNPACKED-LABEL: name: image_load_v2f16_dmask_1000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -540,41 +545,11 @@ define amdgpu_ps <2 x half> @image_load_v2f16_dmask_1000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_v2f16_dmask_1000
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret <2 x half> %tex
}
define amdgpu_ps <2 x half> @image_load_v2f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
- ; UNPACKED-LABEL: name: image_load_v2f16_dmask_0000
- ; UNPACKED: bb.1 (%ir-block.0):
- ; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
- ; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(<2 x f16>) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: $vgpr0 = COPY [[DEF]](<2 x f16>)
- ; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; PACKED-LABEL: name: image_load_v2f16_dmask_0000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -584,42 +559,20 @@ define amdgpu_ps <2 x half> @image_load_v2f16_dmask_0000(<8 x i32> inreg %rsrc,
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(<2 x f16>) = G_IMPLICIT_DEF
; PACKED-NEXT: $vgpr0 = COPY [[DEF]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; UNPACKED-LABEL: name: image_load_v2f16_dmask_0000
+ ; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(<2 x f16>) = G_IMPLICIT_DEF
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[DEF]](<2 x f16>)
+ ; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret <2 x half> %tex
}
define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1100(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
- ; UNPACKED-LABEL: name: image_load_v3f16_dmask_1100
- ; UNPACKED: bb.1 (%ir-block.0):
- ; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
- ; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; UNPACKED-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[UV1]](i32)
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[DEF]](i16)
- ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF1]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BUILD_VECTOR2]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[BUILD_VECTOR3]](<2 x f16>)
- ; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
; PACKED-LABEL: name: image_load_v3f16_dmask_1100
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -637,20 +590,15 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1100(<8 x i32> inreg %rsrc,
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; PACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
; PACKED-NEXT: $vgpr1 = COPY [[BITCAST]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <3 x half> %tex
-}
-
-define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
- ; UNPACKED-LABEL: name: image_load_v3f16_dmask_1000
+ ; UNPACKED-LABEL: name: image_load_v3f16_dmask_1100
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
@@ -666,19 +614,29 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](i32)
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[DEF]](i16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[DEF]](i16)
- ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF1]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BUILD_VECTOR2]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[BUILD_VECTOR3]](<2 x f16>)
+ ; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
+ ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[DEF]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
+ %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <3 x half> %tex
+}
+
+define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; PACKED-LABEL: name: image_load_v3f16_dmask_1000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -696,45 +654,61 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc,
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; PACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
; PACKED-NEXT: $vgpr1 = COPY [[BITCAST]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <3 x half> %tex
-}
-
-define amdgpu_ps <3 x half> @image_load_v3f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
- ; UNPACKED-LABEL: name: image_load_v3f16_dmask_0000
+ ; UNPACKED-LABEL: name: image_load_v3f16_dmask_1000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[DEF]], [[C]]
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY2]](i32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
+ %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <3 x half> %tex
+}
+
+define amdgpu_ps <3 x half> @image_load_v3f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; PACKED-LABEL: name: image_load_v3f16_dmask_0000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; PACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
@@ -742,11 +716,48 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_0000(<8 x i32> inreg %rsrc,
; PACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; PACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+ ; UNPACKED-LABEL: name: image_load_v3f16_dmask_0000
+ ; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY2]](i32)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
+ ; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret <3 x half> %tex
}
define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v4f16_dmask_1110
+ ; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x f16>)
+ ; PACKED-NEXT: $vgpr0 = COPY [[UV]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[UV1]](<2 x f16>)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
; UNPACKED-LABEL: name: image_load_v4f16_dmask_1110
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -780,8 +791,12 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_v4f16_dmask_1110
+ %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <4 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v4f16_dmask_1100
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -797,16 +812,11 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc,
; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x f16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[UV1]](<2 x f16>)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
+ ; PACKED-NEXT: [[DEF:%[0-9]+]]:_(<2 x f16>) = G_IMPLICIT_DEF
+ ; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[DEF]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <4 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v4f16_dmask_1100
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -839,8 +849,12 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_v4f16_dmask_1100
+ %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <4 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v4f16_dmask_1000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -856,16 +870,11 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc,
; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(<2 x f16>) = G_IMPLICIT_DEF
; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
; PACKED-NEXT: $vgpr1 = COPY [[DEF]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <4 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v4f16_dmask_1000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -895,33 +904,21 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_v4f16_dmask_1000
+ %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <4 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_v4f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_v4f16_dmask_0000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x f16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(<2 x f16>) = G_IMPLICIT_DEF
- ; PACKED-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr0 = COPY [[DEF]](<2 x f16>)
; PACKED-NEXT: $vgpr1 = COPY [[DEF]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
- ret <4 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_v4f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v4f16_dmask_0000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -932,22 +929,33 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_0000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: $vgpr0 = COPY [[DEF]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[DEF]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_v4f16_dmask_0000
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[DEF:%[0-9]+]]:_(<2 x f16>) = G_IMPLICIT_DEF
- ; PACKED-NEXT: $vgpr0 = COPY [[DEF]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[DEF]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret <4 x half> %tex
}
define amdgpu_ps half @image_load_tfe_f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_f16_dmask_0000
+ ; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+ ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
+ ; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; PACKED-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
; UNPACKED-LABEL: name: image_load_tfe_f16_dmask_0000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -970,8 +978,15 @@ define amdgpu_ps half @image_load_tfe_f16_dmask_0000(<8 x i32> inreg %rsrc, i32
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: $vgpr0 = COPY [[UV]](i32)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_tfe_f16_dmask_0000
+ %res = call { half, i32 } @llvm.amdgcn.image.load.2d.sl_f16i32s.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { half, i32 } %res, 0
+ %tfe = extractvalue { half, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret half %tex
+}
+
+define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v2f16_dmask_1000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -990,17 +1005,10 @@ define amdgpu_ps half @image_load_tfe_f16_dmask_0000(<8 x i32> inreg %rsrc, i32
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
+ ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[UV]](i32)
; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV]](i32)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- %res = call { half, i32 } @llvm.amdgcn.image.load.2d.sl_f16i32s.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { half, i32 } %res, 0
- %tfe = extractvalue { half, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret half %tex
-}
-
-define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
; UNPACKED-LABEL: name: image_load_tfe_v2f16_dmask_1000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1030,8 +1038,15 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rs
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_tfe_v2f16_dmask_1000
+ %res = call { <2 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v2f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <2 x half>, i32 } %res, 0
+ %tfe = extractvalue { <2 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <2 x half> %tex
+}
+
+define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v2f16_dmask_0000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1054,14 +1069,6 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rs
; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; PACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- %res = call { <2 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v2f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <2 x half>, i32 } %res, 0
- %tfe = extractvalue { <2 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <2 x half> %tex
-}
-
-define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v2f16_dmask_0000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1091,8 +1098,15 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rs
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
- ; PACKED-LABEL: name: image_load_tfe_v2f16_dmask_0000
+ %res = call { <2 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v2f16i32s.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <2 x half>, i32 } %res, 0
+ %tfe = extractvalue { <2 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <2 x half> %tex
+}
+
+define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v3f16_dmask_1100
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1109,20 +1123,22 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rs
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[UV]](i32)
+ ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
+ ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- %res = call { <2 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v2f16i32s.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <2 x half>, i32 } %res, 0
- %tfe = extractvalue { <2 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <2 x half> %tex
-}
-
-define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; PACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[DEF1]], [[C]]
+ ; PACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
+ ; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST2]](<2 x f16>)
+ ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
; UNPACKED-LABEL: name: image_load_tfe_v3f16_dmask_1100
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1142,21 +1158,32 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; UNPACKED-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[UV1]](i32)
- ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; UNPACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[DEF1]](i16)
- ; UNPACKED-NEXT: [[DEF2:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF2]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BUILD_VECTOR2]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[BUILD_VECTOR3]](<2 x f16>)
+ ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[DEF1]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v3f16_dmask_1100
+ %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <3 x half>, i32 } %res, 0
+ %tfe = extractvalue { <3 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <3 x half> %tex
+}
+
+define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v3f16_dmask_1000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1173,33 +1200,22 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rs
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
- ; PACKED-NEXT: [[UV2:%[0-9]+]]:_(i16), [[UV3:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BITCAST]](<2 x i16>)
- ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[UV2]](i16), [[UV3]](i16), [[DEF1]](i16)
- ; PACKED-NEXT: [[UV4:%[0-9]+]]:_(i16), [[UV5:%[0-9]+]]:_(i16), [[UV6:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
- ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[UV4]](i16)
- ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[UV5]](i16)
- ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[UV6]](i16)
- ; PACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: [[UV7:%[0-9]+]]:_(f16), [[UV8:%[0-9]+]]:_(f16), [[UV9:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x f16>)
- ; PACKED-NEXT: [[DEF2:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<4 x f16>) = G_BUILD_VECTOR [[UV7]](f16), [[UV8]](f16), [[UV9]](f16), [[DEF2]](f16)
- ; PACKED-NEXT: [[UV10:%[0-9]+]]:_(<2 x f16>), [[UV11:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<4 x f16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV10]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[UV11]](<2 x f16>)
+ ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; PACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[DEF1]], [[C]]
+ ; PACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
+ ; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST2]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <3 x half>, i32 } %res, 0
- %tfe = extractvalue { <3 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <3 x half> %tex
-}
-
-define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v3f16_dmask_1000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1219,20 +1235,32 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[DEF1]](i16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[DEF1]](i16)
- ; UNPACKED-NEXT: [[DEF2:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF2]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BUILD_VECTOR2]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[BUILD_VECTOR3]](<2 x f16>)
+ ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[DEF1]], [[C]]
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v3f16_dmask_1000
+ %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <3 x half>, i32 } %res, 0
+ %tfe = extractvalue { <3 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <3 x half> %tex
+}
+
+define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v3f16_dmask_0000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1252,30 +1280,19 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rs
; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
- ; PACKED-NEXT: [[UV2:%[0-9]+]]:_(i16), [[UV3:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BITCAST]](<2 x i16>)
- ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[UV2]](i16), [[UV3]](i16), [[DEF1]](i16)
- ; PACKED-NEXT: [[UV4:%[0-9]+]]:_(i16), [[UV5:%[0-9]+]]:_(i16), [[UV6:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
- ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[UV4]](i16)
- ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[UV5]](i16)
- ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[UV6]](i16)
- ; PACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: [[UV7:%[0-9]+]]:_(f16), [[UV8:%[0-9]+]]:_(f16), [[UV9:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x f16>)
- ; PACKED-NEXT: [[DEF2:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<4 x f16>) = G_BUILD_VECTOR [[UV7]](f16), [[UV8]](f16), [[UV9]](f16), [[DEF2]](f16)
- ; PACKED-NEXT: [[UV10:%[0-9]+]]:_(<2 x f16>), [[UV11:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<4 x f16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV10]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[UV11]](<2 x f16>)
+ ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; PACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[DEF1]], [[C]]
+ ; PACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
+ ; PACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST2]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <3 x half>, i32 } %res, 0
- %tfe = extractvalue { <3 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <3 x half> %tex
-}
-
-define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v3f16_dmask_0000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1295,20 +1312,32 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[DEF1]](i16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[DEF1]](i16)
- ; UNPACKED-NEXT: [[DEF2:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF2]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BUILD_VECTOR2]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[BUILD_VECTOR3]](<2 x f16>)
+ ; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[DEF1]], [[C]]
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v3f16_dmask_0000
+ %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <3 x half>, i32 } %res, 0
+ %tfe = extractvalue { <3 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <3 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_1110
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1325,33 +1354,16 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rs
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
- ; PACKED-NEXT: [[UV2:%[0-9]+]]:_(i16), [[UV3:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BITCAST]](<2 x i16>)
- ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[UV2]](i16), [[UV3]](i16), [[DEF1]](i16)
- ; PACKED-NEXT: [[UV4:%[0-9]+]]:_(i16), [[UV5:%[0-9]+]]:_(i16), [[UV6:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
- ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[UV4]](i16)
- ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[UV5]](i16)
- ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[UV6]](i16)
- ; PACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: [[UV7:%[0-9]+]]:_(f16), [[UV8:%[0-9]+]]:_(f16), [[UV9:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x f16>)
- ; PACKED-NEXT: [[DEF2:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<4 x f16>) = G_BUILD_VECTOR [[UV7]](f16), [[UV8]](f16), [[UV9]](f16), [[DEF2]](f16)
- ; PACKED-NEXT: [[UV10:%[0-9]+]]:_(<2 x f16>), [[UV11:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<4 x f16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[UV10]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[UV11]](<2 x f16>)
+ ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV1]](i32)
+ ; PACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST1]](<2 x i16>)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST3]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %res = call { <3 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v3f16i32s.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <3 x half>, i32 } %res, 0
- %tfe = extractvalue { <3 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <3 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v4f16_dmask_1110
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1387,8 +1399,15 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rs
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_1110
+ %res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <4 x half>, i32 } %res, 0
+ %tfe = extractvalue { <4 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <4 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_1100
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1405,24 +1424,16 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rs
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
+ ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
- ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV1]](i32)
- ; PACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
- ; PACKED-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST1]](<2 x i16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST3]](<2 x f16>)
+ ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i16>) = G_IMPLICIT_DEF
+ ; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[DEF1]](<2 x i16>)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
+ ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST2]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <4 x half>, i32 } %res, 0
- %tfe = extractvalue { <4 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <4 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v4f16_dmask_1100
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1457,8 +1468,15 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_1100
+ %res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <4 x half>, i32 } %res, 0
+ %tfe = extractvalue { <4 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <4 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_1000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1475,7 +1493,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
+ ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i16>) = G_IMPLICIT_DEF
@@ -1485,14 +1503,6 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
; PACKED-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
; PACKED-NEXT: $vgpr1 = COPY [[BITCAST2]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <4 x half>, i32 } %res, 0
- %tfe = extractvalue { <4 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <4 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v4f16_dmask_1000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1525,8 +1535,15 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rs
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_1000
+ %res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+ %tex = extractvalue { <4 x half>, i32 } %res, 0
+ %tfe = extractvalue { <4 x half>, i32 } %res, 1
+ store i32 %tfe, ptr addrspace(1) poison
+ ret <4 x half> %tex
+}
+
+define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+ ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_0000
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
@@ -1553,14 +1570,6 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rs
; PACKED-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
; PACKED-NEXT: $vgpr1 = COPY [[BITCAST2]](<2 x f16>)
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- %res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
- %tex = extractvalue { <4 x half>, i32 } %res, 0
- %tfe = extractvalue { <4 x half>, i32 } %res, 1
- store i32 %tfe, ptr addrspace(1) poison
- ret <4 x half> %tex
-}
-
-define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_tfe_v4f16_dmask_0000
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
@@ -1593,34 +1602,6 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rs
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
; UNPACKED-NEXT: $vgpr1 = COPY [[BITCAST1]](<2 x f16>)
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
- ;
- ; PACKED-LABEL: name: image_load_tfe_v4f16_dmask_0000
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
- ; PACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
- ; PACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; PACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[UV]](i32)
- ; PACKED-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i16>) = G_IMPLICIT_DEF
- ; PACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; PACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[BITCAST]](<2 x i16>)
- ; PACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[DEF1]](<2 x i16>)
- ; PACKED-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
- ; PACKED-NEXT: $vgpr1 = COPY [[BITCAST2]](<2 x f16>)
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%res = call { <4 x half>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f16i32s.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
%tex = extractvalue { <4 x half>, i32 } %res, 0
%tfe = extractvalue { <4 x half>, i32 } %res, 1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
index 49ddf6d60dd40..61006f8c1eab6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -24,10 +24,9 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -55,10 +54,9 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -153,12 +151,10 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -186,12 +182,10 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -290,17 +284,14 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -329,17 +320,14 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -448,17 +436,14 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -487,17 +472,14 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -606,12 +588,10 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -639,12 +619,10 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -743,17 +721,14 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -782,17 +757,14 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -902,12 +874,11 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -936,12 +907,11 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1040,14 +1010,12 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1076,14 +1044,12 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1185,12 +1151,10 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1218,12 +1182,10 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1322,17 +1284,14 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -1361,17 +1320,14 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1481,14 +1437,12 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1517,14 +1471,12 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1627,19 +1579,16 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1668,19 +1617,16 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1792,14 +1738,12 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -1828,14 +1772,12 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1938,17 +1880,14 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -1977,17 +1916,14 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2096,17 +2032,15 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2134,17 +2068,15 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2250,20 +2182,17 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2291,20 +2220,17 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2416,17 +2342,14 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -2455,17 +2378,14 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2574,21 +2494,17 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -2617,21 +2533,17 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2748,20 +2660,17 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2789,20 +2698,17 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2914,24 +2820,20 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2959,24 +2861,20 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[TRUNC2]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3096,18 +2994,15 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3136,18 +3031,15 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3258,26 +3150,20 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3306,26 +3192,20 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3452,39 +3332,30 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX9-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
+ ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX9-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX9-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX9-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC8:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[TRUNC4]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC5]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[TRUNC7]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC8]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3513,39 +3384,30 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
+ ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[TRUNC4]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC5]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[TRUNC7]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC8]](f16), [[DEF]](f16)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3702,20 +3564,17 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3744,20 +3603,17 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3872,28 +3728,22 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3922,28 +3772,22 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4073,21 +3917,17 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -4116,21 +3956,17 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4247,31 +4083,24 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -4300,31 +4129,24 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4462,23 +4284,19 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4507,23 +4325,19 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4644,33 +4458,26 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX9-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<10 x f16>) = G_CONCAT_VECTORS [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<10 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[CONCAT_VECTORS]](<10 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4699,33 +4506,26 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4866,18 +4666,15 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -4906,18 +4703,15 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5028,26 +4822,20 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -5076,26 +4864,20 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5223,20 +5005,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5265,20 +5044,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5393,28 +5169,22 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5443,28 +5213,22 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5594,21 +5358,17 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -5637,21 +5397,17 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5768,31 +5524,24 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -5821,31 +5570,24 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5983,23 +5725,19 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6028,23 +5766,19 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6165,33 +5899,26 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX9-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<10 x f16>) = G_CONCAT_VECTORS [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<10 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[CONCAT_VECTORS]](<10 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6220,33 +5947,26 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6387,12 +6107,10 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6420,12 +6138,10 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6524,17 +6240,14 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -6563,17 +6276,14 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6683,14 +6393,12 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6719,14 +6427,12 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6829,19 +6535,16 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6870,19 +6573,16 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6994,10 +6694,9 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7025,10 +6724,9 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7123,12 +6821,10 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7156,12 +6852,10 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7261,12 +6955,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7295,12 +6988,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7399,14 +7091,12 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7435,14 +7125,12 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7546,34 +7234,27 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX9-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX9-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
; GFX9-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX9-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -7599,34 +7280,27 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
; GFX10-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -7763,34 +7437,27 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX9-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX9-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX9-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX9-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX9-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX9-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX9-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX9-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX9-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7818,34 +7485,27 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
index 6898a42cdab6a..4934f93ac18df 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -23,18 +23,15 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -145,26 +142,20 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -291,39 +282,30 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
+ ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[TRUNC4]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC5]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[TRUNC7]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC8]](f16), [[DEF]](f16)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
index 41b66446f0270..2ab7e0d348f41 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -23,17 +23,15 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -139,24 +137,20 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -276,35 +270,29 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[TRUNC4]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC5]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<14 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -450,18 +438,16 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -572,25 +558,21 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -715,19 +697,17 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -837,26 +817,22 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -982,20 +958,18 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1111,27 +1085,23 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1262,17 +1232,15 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1378,24 +1346,20 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1516,18 +1480,16 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1638,25 +1600,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1781,19 +1739,17 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1903,26 +1859,22 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2048,20 +2000,18 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2177,27 +2127,23 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2330,28 +2276,24 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<14 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
; GFX10-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -2476,28 +2418,24 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<14 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
index f511e3b7df5b6..0bfbcbf770e9c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX81 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX81 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -23,10 +23,9 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY10]](i32)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
+ ; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; GFX81-LABEL: name: image_store_f16
@@ -45,10 +44,9 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; GFX81-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX81-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX81-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX81-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX81-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY10]](i32)
; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
+ ; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX81-NEXT: S_ENDPGM 0
;
; GFX9-LABEL: name: image_store_f16
@@ -67,33 +65,11 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX9-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY10]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
+ ; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX9-NEXT: S_ENDPGM 0
;
- ; GFX10-LABEL: name: image_store_f16
- ; GFX10: bb.1 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; GFX10-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; GFX10-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; GFX10-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
- ; GFX10-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
- ; GFX10-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
- ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
- ; GFX10-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX10-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX10-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX10-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[COPY8]](i32), [[COPY9]](i32), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
; GFX12-LABEL: name: image_store_f16
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
@@ -247,21 +223,13 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr3
- ; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY10]](<2 x f16>), [[COPY11]](<2 x f16>)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x f16>)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
- ; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x i16>)
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
- ; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
- ; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
- ; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR4]](<3 x i32>), 7, [[BUILD_VECTOR2]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x f16>)
+ ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
+ ; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<3 x i32>), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; GFX81-LABEL: name: image_store_v3f16
@@ -281,20 +249,26 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; GFX81-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX81-NEXT: [[COPY10:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
; GFX81-NEXT: [[COPY11:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr3
- ; GFX81-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY10]](<2 x f16>), [[COPY11]](<2 x f16>)
- ; GFX81-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
- ; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
- ; GFX81-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX81-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x f16>)
- ; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
- ; GFX81-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
- ; GFX81-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
- ; GFX81-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
- ; GFX81-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x i16>)
- ; GFX81-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX81-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<6 x i16>) = G_BUILD_VECTOR [[UV7]](i16), [[UV8]](i16), [[UV9]](i16), [[DEF]](i16), [[DEF]](i16), [[DEF]](i16)
- ; GFX81-NEXT: [[BITCAST3:%[0-9]+]]:_(<3 x i32>) = G_BITCAST [[BUILD_VECTOR4]](<6 x i16>)
- ; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST3]](<3 x i32>), 7, [[BUILD_VECTOR2]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
+ ; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x f16>)
+ ; GFX81-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX81-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX81-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x f16>)
+ ; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX81-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX81-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
+ ; GFX81-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX81-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; GFX81-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
+ ; GFX81-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
+ ; GFX81-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX81-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; GFX81-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
+ ; GFX81-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
+ ; GFX81-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; GFX81-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
+ ; GFX81-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x i16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
+ ; GFX81-NEXT: [[BITCAST5:%[0-9]+]]:_(<3 x i32>) = G_BITCAST [[CONCAT_VECTORS]](<6 x i16>)
+ ; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST5]](<3 x i32>), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
; GFX81-NEXT: S_ENDPGM 0
;
; GFX9-LABEL: name: image_store_v3f16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
index eaa0bb5ed3066..ac1b50a967ebe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
declare half @llvm.amdgcn.fract.f16(half)
declare float @llvm.amdgcn.fract.f32(float)
@@ -9,8 +9,11 @@ define amdgpu_ps half @s_fract_f16(half inreg %src) {
; GFX12-LABEL: s_fract_f16:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_fract_f16_e32 v0.l, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.l
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_2)
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: s_add_f16 s0, s0, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
%fract = call half @llvm.amdgcn.fract.f16(half %src)
%res = fadd half %fract, %fract
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.frexp.mant.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.frexp.mant.ll
index 5e66ee182ed25..40de8f5cd6623 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.frexp.mant.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.frexp.mant.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
declare half @llvm.amdgcn.frexp.mant.f16(half)
declare float @llvm.amdgcn.frexp.mant.f32(float)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index cb52ce10c30bf..72645e5d624fb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -1,74 +1,76 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX10NSA %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX10NSA %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -o - %s | FileCheck -check-prefix=GFX10NSA %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX10NSA %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX12 %s
define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
; GFX9-LABEL: gather4_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v0, v1, v0, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4 v[0:3], v0, s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4 v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4 v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -79,68 +81,76 @@ main_body:
define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %face) {
; GFX9-LABEL: gather4_cube:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v1, v1, v0, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4 v[0:3], v[1:2], s[4:11], s[12:15] dmask:0x1 a16 da
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_cube:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_cube:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4 v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -151,68 +161,76 @@ main_body:
define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %slice) {
; GFX9-LABEL: gather4_2darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v1, v1, v0, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4 v[0:3], v[1:2], s[4:11], s[12:15] dmask:0x1 a16 da
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_2darray:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_2darray:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4 v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4 v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -223,68 +241,70 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
; GFX9-LABEL: gather4_c_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4_c v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4_c v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_c_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4_c v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -295,68 +315,76 @@ main_body:
define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %clamp) {
; GFX9-LABEL: gather4_cl_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v1, v1, v0, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4_cl v[0:3], v[1:2], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_cl_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4_cl v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_cl_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_cl v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4_cl v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -367,70 +395,76 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %clamp) {
; GFX9-LABEL: gather4_c_cl_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v4, v2, v1, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4_c_cl v[0:3], v[3:5], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_cl_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_c_cl_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -441,68 +475,76 @@ main_body:
define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, half %s, half %t) {
; GFX9-LABEL: gather4_b_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4_b v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_b_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4_b v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_b_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_b v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4_b v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -513,68 +555,76 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, float %zcompare, half %s, half %t) {
; GFX9-LABEL: gather4_c_b_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v2, v3, v2, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4_c_b v[0:3], v[0:2], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_b_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4_c_b v[0:3], v[0:2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_c_b_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c_b v[0:3], [v0, v1, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4_c_b v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -585,70 +635,82 @@ main_body:
define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, half %s, half %t, half %clamp) {
; GFX9-LABEL: gather4_b_cl_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v4, v2, v1, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4_b_cl v[0:3], v[3:5], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_b_cl_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_b_cl_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -659,71 +721,82 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, float %zcompare, half %s, half %t, half %clamp) {
; GFX9-LABEL: gather4_c_b_cl_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_mov_b64 s[14:15], exec
+; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: v_mov_b32_e32 v7, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, v1
-; GFX9-NEXT: v_mov_b32_e32 v4, v0
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: v_perm_b32 v6, v3, v2, s2
-; GFX9-NEXT: s_and_b64 exec, exec, s[0:1]
-; GFX9-NEXT: image_gather4_c_b_cl v[0:3], v[4:7], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT: image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_b_cl_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s0, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX10NSA-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10NSA-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT: image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_c_b_cl_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, s2
; GFX12-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX12-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX12-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -734,59 +807,67 @@ main_body:
define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %lod) {
; GFX9-LABEL: gather4_l_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: v_perm_b32 v1, v1, v0, s0
-; GFX9-NEXT: image_gather4_l v[0:3], v[1:2], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: s_mov_b32 s0, s2
+; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_l_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: image_gather4_l v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_l_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_l v[0:3], [v0, v2], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: s_mov_b32 s0, s2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: image_gather4_l v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -797,61 +878,67 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %lod) {
; GFX9-LABEL: gather4_c_l_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: v_perm_b32 v4, v2, v1, s0
-; GFX9-NEXT: image_gather4_c_l v[0:3], v[3:5], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: s_mov_b32 s0, s2
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_l_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: image_gather4_c_l v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_c_l_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_c_l v[0:3], [v0, v1, v3], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-NEXT: s_mov_b32 s0, s2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: image_gather4_c_l v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -862,59 +949,61 @@ main_body:
define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
; GFX9-LABEL: gather4_lz_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX9-NEXT: image_gather4_lz v[0:3], v0, s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: s_mov_b32 s0, s2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_lz_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: image_gather4_lz v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_lz_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_lz v[0:3], v0, s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: s_mov_b32 s0, s2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -925,59 +1014,61 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
; GFX9-LABEL: gather4_c_lz_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: s_mov_b32 s15, s13
-; GFX9-NEXT: s_mov_b32 s14, s12
-; GFX9-NEXT: s_mov_b32 s13, s11
-; GFX9-NEXT: s_mov_b32 s12, s10
-; GFX9-NEXT: s_mov_b32 s11, s9
-; GFX9-NEXT: s_mov_b32 s10, s8
-; GFX9-NEXT: s_mov_b32 s9, s7
-; GFX9-NEXT: s_mov_b32 s8, s6
-; GFX9-NEXT: s_mov_b32 s7, s5
-; GFX9-NEXT: s_mov_b32 s6, s4
-; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s0
-; GFX9-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 a16
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: s_mov_b32 s0, s2
+; GFX9-NEXT: s_mov_b32 s1, s3
+; GFX9-NEXT: s_mov_b32 s2, s4
+; GFX9-NEXT: s_mov_b32 s3, s5
+; GFX9-NEXT: s_mov_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s5, s7
+; GFX9-NEXT: s_mov_b32 s6, s8
+; GFX9-NEXT: s_mov_b32 s7, s9
+; GFX9-NEXT: s_mov_b32 s8, s10
+; GFX9-NEXT: s_mov_b32 s9, s11
+; GFX9-NEXT: s_mov_b32 s10, s12
+; GFX9-NEXT: s_mov_b32 s11, s13
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_lz_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10NSA-NEXT: s_mov_b32 s15, s13
-; GFX10NSA-NEXT: s_mov_b32 s14, s12
-; GFX10NSA-NEXT: s_mov_b32 s13, s11
-; GFX10NSA-NEXT: s_mov_b32 s12, s10
-; GFX10NSA-NEXT: s_mov_b32 s11, s9
-; GFX10NSA-NEXT: s_mov_b32 s10, s8
-; GFX10NSA-NEXT: s_mov_b32 s9, s7
-; GFX10NSA-NEXT: s_mov_b32 s8, s6
-; GFX10NSA-NEXT: s_mov_b32 s7, s5
-; GFX10NSA-NEXT: s_mov_b32 s6, s4
-; GFX10NSA-NEXT: s_mov_b32 s5, s3
-; GFX10NSA-NEXT: s_mov_b32 s4, s2
-; GFX10NSA-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: s_mov_b32 s0, s2
+; GFX10NSA-NEXT: s_mov_b32 s1, s3
+; GFX10NSA-NEXT: s_mov_b32 s2, s4
+; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: s_mov_b32 s4, s6
+; GFX10NSA-NEXT: s_mov_b32 s5, s7
+; GFX10NSA-NEXT: s_mov_b32 s6, s8
+; GFX10NSA-NEXT: s_mov_b32 s7, s9
+; GFX10NSA-NEXT: s_mov_b32 s8, s10
+; GFX10NSA-NEXT: s_mov_b32 s9, s11
+; GFX10NSA-NEXT: s_mov_b32 s10, s12
+; GFX10NSA-NEXT: s_mov_b32 s11, s13
+; GFX10NSA-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
; GFX10NSA-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: gather4_c_lz_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: s_mov_b32 s11, s9
-; GFX12-NEXT: s_mov_b32 s10, s8
-; GFX12-NEXT: s_mov_b32 s9, s7
-; GFX12-NEXT: s_mov_b32 s8, s6
-; GFX12-NEXT: s_mov_b32 s7, s5
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s5, s3
-; GFX12-NEXT: s_mov_b32 s4, s2
-; GFX12-NEXT: image_gather4_c_lz v[0:3], [v0, v1], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: s_mov_b32 s0, s2
+; GFX12-NEXT: s_mov_b32 s1, s3
+; GFX12-NEXT: s_mov_b32 s2, s4
+; GFX12-NEXT: s_mov_b32 s3, s5
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: s_mov_b32 s4, s6
+; GFX12-NEXT: s_mov_b32 s5, s7
+; GFX12-NEXT: s_mov_b32 s6, s8
+; GFX12-NEXT: s_mov_b32 s7, s9
+; GFX12-NEXT: s_mov_b32 s8, s10
+; GFX12-NEXT: s_mov_b32 s9, s11
+; GFX12-NEXT: s_mov_b32 s10, s12
+; GFX12-NEXT: s_mov_b32 s11, s13
+; GFX12-NEXT: image_gather4_c_lz v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
index c9f2337061143..c417d090efdbe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
@@ -1,9 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_cd_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -15,9 +19,11 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_cd_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10-NEXT: image_sample_cd_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: image_sample_cd_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
main_body:
@@ -28,6 +34,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_cd_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -39,9 +49,11 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_cd_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
main_body:
@@ -52,6 +64,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -63,9 +79,11 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_cd_cl_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
main_body:
@@ -76,6 +94,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -89,8 +111,10 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_perm_b32 v4, v4, v3, 0x5040100
-; GFX10-NEXT: v_perm_b32 v3, v8, v1, 0x5040100
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
index 400ca7ca664e4..9225426521a11 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
@@ -1,23 +1,35 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12 %s
define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_d_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_1d:
; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_1d:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -29,25 +41,31 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_d_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_2d:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -58,27 +76,43 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %drdh, half %dsdv, half %dtdv, half %drdv, float %s, float %t, float %r) {
; GFX10-LABEL: sample_d_3d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v9, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
-; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x5040100
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v9, 0xffff, v2
+; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX10-NEXT: v_lshl_or_b32 v2, v1, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v9
+; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v10
+; GFX10-NEXT: v_lshl_or_b32 v5, s0, 16, v5
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[2:8], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_3d:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX11-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX11-NEXT: v_lshl_or_b32 v3, s0, 16, v5
+; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_3d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[5:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX12-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX12-NEXT: v_lshl_or_b32 v5, s0, 16, v5
+; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -89,18 +123,30 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_d_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_1d:
; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX11-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_1d:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX12-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -112,25 +158,31 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_d_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_2d:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX11-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX11-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX12-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -141,18 +193,30 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_cl_1d:
; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_cl_1d:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v2, v3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -164,25 +228,31 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_d_cl_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_cl_2d:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX11-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_cl_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX12-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX12-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -193,18 +263,30 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_cl_1d:
; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX11-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX11-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_cl_1d:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX12-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v2, v[3:4]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -218,25 +300,31 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_perm_b32 v4, v4, v3, 0x5040100
-; GFX10-NEXT: v_perm_b32 v3, v8, v1, 0x5040100
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_cl_2d:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX11-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v5, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX11-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v2, v5, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_cl_2d:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX12-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[5:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX12-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX12-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v2, v[5:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -247,28 +335,34 @@ main_body:
define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V1:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v9, v3
-; GFX10-NEXT: v_mov_b32_e32 v10, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
+; GFX10-NEXT: v_mov_b32_e32 v9, v2
+; GFX10-NEXT: v_mov_b32_e32 v10, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX10-NEXT: v_perm_b32 v4, v9, v10, 0x5040100
+; GFX10-NEXT: v_mov_b32_e32 v3, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
; GFX10-NEXT: image_sample_c_d_o_g16 v0, v[2:8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_o_2darray_V1:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-NEXT: v_lshl_or_b32 v3, v5, 16, v4
+; GFX11-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_o_2darray_V1:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-NEXT: v_lshl_or_b32 v5, v5, 16, v4
; GFX12-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -280,28 +374,34 @@ main_body:
define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V2:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v9, v3
-; GFX10-NEXT: v_mov_b32_e32 v10, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
+; GFX10-NEXT: v_mov_b32_e32 v9, v2
+; GFX10-NEXT: v_mov_b32_e32 v10, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX10-NEXT: v_perm_b32 v4, v9, v10, 0x5040100
+; GFX10-NEXT: v_mov_b32_e32 v3, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
+; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
; GFX10-NEXT: image_sample_c_d_o_g16 v[0:1], v[2:8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_o_2darray_V2:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
-; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX11-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-NEXT: v_lshl_or_b32 v3, v5, 16, v4
+; GFX11-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_o_2darray_V2:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-NEXT: v_lshl_or_b32 v5, v5, 16, v4
; GFX12-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
index 8d92af391d89d..a6d95b2c4c5b1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
@@ -9,28 +9,28 @@
define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, half %data) {
; UNPACKED-LABEL: image_store_f16:
; UNPACKED: ; %bb.0:
-; UNPACKED-NEXT: s_mov_b32 s11, s9
-; UNPACKED-NEXT: s_mov_b32 s10, s8
-; UNPACKED-NEXT: s_mov_b32 s9, s7
-; UNPACKED-NEXT: s_mov_b32 s8, s6
-; UNPACKED-NEXT: s_mov_b32 s7, s5
-; UNPACKED-NEXT: s_mov_b32 s6, s4
-; UNPACKED-NEXT: s_mov_b32 s5, s3
-; UNPACKED-NEXT: s_mov_b32 s4, s2
-; UNPACKED-NEXT: image_store v2, v[0:1], s[4:11] dmask:0x1 unorm d16
+; UNPACKED-NEXT: s_mov_b32 s0, s2
+; UNPACKED-NEXT: s_mov_b32 s1, s3
+; UNPACKED-NEXT: s_mov_b32 s2, s4
+; UNPACKED-NEXT: s_mov_b32 s3, s5
+; UNPACKED-NEXT: s_mov_b32 s4, s6
+; UNPACKED-NEXT: s_mov_b32 s5, s7
+; UNPACKED-NEXT: s_mov_b32 s6, s8
+; UNPACKED-NEXT: s_mov_b32 s7, s9
+; UNPACKED-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 unorm d16
; UNPACKED-NEXT: s_endpgm
;
; GFX81-LABEL: image_store_f16:
; GFX81: ; %bb.0:
-; GFX81-NEXT: s_mov_b32 s11, s9
-; GFX81-NEXT: s_mov_b32 s10, s8
-; GFX81-NEXT: s_mov_b32 s9, s7
-; GFX81-NEXT: s_mov_b32 s8, s6
-; GFX81-NEXT: s_mov_b32 s7, s5
-; GFX81-NEXT: s_mov_b32 s6, s4
-; GFX81-NEXT: s_mov_b32 s5, s3
-; GFX81-NEXT: s_mov_b32 s4, s2
-; GFX81-NEXT: image_store v2, v[0:1], s[4:11] dmask:0x1 unorm d16
+; GFX81-NEXT: s_mov_b32 s0, s2
+; GFX81-NEXT: s_mov_b32 s1, s3
+; GFX81-NEXT: s_mov_b32 s2, s4
+; GFX81-NEXT: s_mov_b32 s3, s5
+; GFX81-NEXT: s_mov_b32 s4, s6
+; GFX81-NEXT: s_mov_b32 s5, s7
+; GFX81-NEXT: s_mov_b32 s6, s8
+; GFX81-NEXT: s_mov_b32 s7, s9
+; GFX81-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 unorm d16
; GFX81-NEXT: s_endpgm
call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index 1e41dc92ebe9d..ea69f58ead7d4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1030 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.13 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1013 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
-; RUN: not llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.12 < %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
+; RUN: not llc -global-isel -mtriple=amdgpu10.12 < %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
; uint4 llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(uint node_ptr, float ray_extent, float3 ray_origin, float3 ray_dir, float3 ray_inv_dir, uint4 texture_descr)
; uint4 llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(uint node_ptr, float ray_extent, float3 ray_origin, half3 ray_dir, half3 ray_inv_dir, uint4 texture_descr)
@@ -71,9 +71,12 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
;
; GFX11-FAKE16-LABEL: image_bvh_intersect_ray_a16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_perm_b32 v9, v7, v5, 0x7060302
-; GFX11-FAKE16-NEXT: v_perm_b32 v10, v8, v6, 0x5040100
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v5, 0x5040100
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v8, v5, 16, v9
+; GFX11-FAKE16-NEXT: v_perm_b32 v9, v5, v7, 0x7060302
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v6, 16, v10
; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v0, v1, v[2:4], v[8:10]], s[0:3] a16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
@@ -137,9 +140,12 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
;
; GFX11-FAKE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_perm_b32 v10, v8, v6, 0x7060302
-; GFX11-FAKE16-NEXT: v_perm_b32 v11, v9, v7, 0x5040100
-; GFX11-FAKE16-NEXT: v_perm_b32 v9, v8, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff, v9
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v6, 16, v10
+; GFX11-FAKE16-NEXT: v_perm_b32 v10, v6, v8, 0x7060302
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v7, 16, v11
; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[0:1], v2, v[3:5], v[9:11]], s[0:3] a16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
@@ -361,33 +367,37 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
;
; GFX11-FAKE16-LABEL: image_bvh_intersect_ray_a16_vgpr_descr:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v4 :: v_dual_mov_b32 v16, v3
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, v2 :: v_dual_mov_b32 v14, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v13, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v19, v7, v5, 0x7060302
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v8, v6, 0x5040100
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v7, v5, 0x5040100
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, v4 :: v_dual_and_b32 v0, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v14, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v5, 16, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v7, 0x7060302
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v6, 16, v1
; GFX11-FAKE16-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v9
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v10
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v11
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v12
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v10
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v11
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v12
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[11:12]
-; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v14, v13, v[15:17], v[18:20]], s[0:3] a16
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11_vgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr15_vgpr16_vgpr17
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr18_vgpr19_vgpr20
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[11:12]
+; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v16, v17, v[13:15], v[4:6]], s[4:7] a16
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr11_vgpr12
+; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
@@ -613,33 +623,38 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
;
; GFX11-FAKE16-LABEL: image_bvh64_intersect_ray_a16_vgpr_descr:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, v5 :: v_dual_mov_b32 v18, v4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v14, v2
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, v1 :: v_dual_mov_b32 v15, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v21, v8, v6, 0x7060302
-; GFX11-FAKE16-NEXT: v_perm_b32 v22, v9, v7, 0x5040100
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v8, v6, 0x5040100
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, v2 :: v_dual_mov_b32 v14, v3
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, v4 :: v_dual_mov_b32 v16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v6, 16, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v6, v8, 0x7060302
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v7, 16, v1
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-FAKE16-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v10
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v11
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v12
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v13
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v10
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v11
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v12
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v13
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
-; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[15:16], v14, v[17:19], v[20:22]], s[0:3] a16
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr15_vgpr16
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr17_vgpr18_vgpr19
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr20_vgpr21_vgpr22
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[10:11]
+; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[12:13]
+; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[17:18], v19, v[14:16], v[4:6]], s[4:7] a16
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr17_vgpr18
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr19
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr14_vgpr15_vgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr12_vgpr13
+; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f16(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
@@ -810,29 +825,71 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
;
-; GFX11-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: v_dual_mov_b32 v4, 1.0 :: v_dual_mov_b32 v5, 2.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, s0, s0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX11-NEXT: v_add_co_u32 v2, s0, s2, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
-; GFX11-NEXT: flat_load_b32 v6, v[0:1]
-; GFX11-NEXT: flat_load_b32 v7, v[2:3]
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x46004200
-; GFX11-NEXT: v_mov_b32_e32 v1, 0x47004400
-; GFX11-NEXT: v_dual_mov_b32 v2, 0x48004500 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[3:5], v[0:2]], s[4:7] a16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_movk_i32 s8, 0x4600
+; GFX11-TRUE16-NEXT: s_movk_i32 s9, 0x4700
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s8, s8, 0x4200
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s9, s9, 0x4400
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v4, 2, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 2.0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_movk_i32 s3, 0x4800
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 1.0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s10, s3, 0x4500
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v7, v[2:3]
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v5, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s8 :: v_dual_mov_b32 v4, s9
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[4:7] a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0x42004600
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, 0x44004700
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0x45004800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s10
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v4, 2, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, 2.0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, 1.0
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-FAKE16-NEXT: flat_load_b32 v6, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v7, v[2:3]
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s9
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[4:7] a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX11-FAKE16-NEXT: s_endpgm
%lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep_node_ptr = getelementptr inbounds i32, ptr %p_node_ptr, i32 %lid
%node_ptr = load i32, ptr %gep_node_ptr, align 4
@@ -1012,31 +1069,70 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
;
-; GFX11-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: v_mov_b32_e32 v2, 0x48004500
-; GFX11-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX11-NEXT: v_mov_b32_e32 v6, 0xb36211c6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: v_bfrev_b32_e32 v7, 4.0
-; GFX11-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, s4, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
-; GFX11-NEXT: flat_load_b32 v8, v[0:1]
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x46004200
-; GFX11-NEXT: v_mov_b32_e32 v1, 0x47004400
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[3:5], v[0:2]], s[0:3] a16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_movk_i32 s8, 0x4600
+; GFX11-TRUE16-NEXT: s_movk_i32 s9, 0x4700
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 1.0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s8, s8, 0x4200
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s9, s9, 0x4400
+; GFX11-TRUE16-NEXT: v_bfrev_b32_e32 v7, 4.0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s6
+; GFX11-TRUE16-NEXT: s_movk_i32 s7, 0x4800
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, 2.0
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s10, s7, 0x4500
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, 0xb36211c6 :: v_dual_mov_b32 v5, s10
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-TRUE16-NEXT: flat_load_b32 v8, v[0:1]
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s9
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[0:3] a16
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 1.0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0x42004600
+; GFX11-FAKE16-NEXT: s_mov_b32 s9, 0x44004700
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0x45004800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, 0xb36211c6 :: v_dual_mov_b32 v5, s10
+; GFX11-FAKE16-NEXT: v_bfrev_b32_e32 v7, 4.0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s9 :: v_dual_mov_b32 v3, s8
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s6
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, 2.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s6
+; GFX11-FAKE16-NEXT: flat_load_b32 v8, v[0:1]
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[0:3] a16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX11-FAKE16-NEXT: s_endpgm
%lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
%ray_extent = load float, ptr %gep_ray, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
index 302cf1d691436..fd3e46ecd8615 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
@@ -1,12 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
+; UNSUPPORTED: true
+; RUN: llc -global-isel -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
+; RUN: llc -global-isel -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
+; RUN: llc -global-isel -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
+; RUN: llc -global-isel -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
+; RUN: llc -global-isel -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
+; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
+; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
+; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
; GFX67-LABEL: name: raw_buffer_load_i8_tfe
@@ -321,150 +322,146 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX67: bb.1 (%ir-block.0):
; GFX67-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX67-NEXT: {{ $}}
- ; GFX67-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX67-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX67-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX67-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX67-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX67-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX67-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX67-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX67-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX67-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX67-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX67-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX67-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX67-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX67-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX67-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX67-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX67-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX67-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX67-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX67-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX67-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX67-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX67-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX67-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX67-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX67-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX67-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX67-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX67-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX67-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX67-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX67-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
+ ; GFX67-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
+ ; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
+ ; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE3]], %subreg.sub2_sub3
+ ; GFX67-NEXT: BUFFER_STORE_SHORT_ADDR64 [[COPY8]], [[REG_SEQUENCE1]], [[REG_SEQUENCE4]], 0, 0, 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
+ ; GFX67-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_1]], %subreg.sub0_sub1, [[REG_SEQUENCE5]], %subreg.sub2_sub3
+ ; GFX67-NEXT: BUFFER_STORE_DWORD_ADDR64 [[COPY9]], [[REG_SEQUENCE2]], [[REG_SEQUENCE6]], 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX67-NEXT: S_ENDPGM 0
;
; GFX8-LABEL: name: raw_buffer_load_f16_tfe
; GFX8: bb.1 (%ir-block.0):
; GFX8-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX8-NEXT: {{ $}}
- ; GFX8-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX8-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX8-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX8-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX8-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX8-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX8-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX8-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX8-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX8-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX8-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX8-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX8-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX8-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX8-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
+ ; GFX8-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
+ ; GFX8-NEXT: FLAT_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
; GFX910-LABEL: name: raw_buffer_load_f16_tfe
; GFX910: bb.1 (%ir-block.0):
; GFX910-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX910-NEXT: {{ $}}
- ; GFX910-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX910-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX910-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX910-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX910-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX910-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX910-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX910-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX910-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX910-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX910-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX910-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX910-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX910-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX910-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX910-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX910-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX910-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX910-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX910-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX910-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX910-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX910-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX910-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX910-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX910-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX910-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX910-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX910-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX910-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX910-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX910-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX910-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
+ ; GFX910-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
+ ; GFX910-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
; GFX11-LABEL: name: raw_buffer_load_f16_tfe
; GFX11: bb.1 (%ir-block.0):
; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX11-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX11-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX11-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX11-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX11-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
+ ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
+ ; GFX11-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
; GFX1200: bb.1 (%ir-block.0):
; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX1200-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX1200-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX1200-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX1200-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1200-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX1200-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX1200-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1200-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX1200-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX1200-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1200-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
; GFX1250: bb.1 (%ir-block.0):
; GFX1250-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX1250-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX1250-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX1250-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX1250-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+ ; GFX1250-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { half, i32 } @llvm.amdgcn.raw.buffer.load.sl_f16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
%data = extractvalue { half, i32 } %res, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index 71cc000a63703..ff59ee2432569 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 %voffset, i32 inreg %soffset) {
@@ -8,38 +8,30 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
@@ -67,36 +59,28 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
index 9e630839861b2..a9ada36d4b2ff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1200 %s
; RUN: llc -global-isel -mtriple=amdgpu12.50-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1250 %s
; FIXME: Test with SI when argument lowering not broken for f16
@@ -947,19 +947,15 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
; GFX8: bb.1 (%ir-block.0):
; GFX8-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
- ; GFX8-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX8-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX8-NEXT: G_AMDGPU_BUFFER_STORE_SHORT [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX8-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX8-NEXT: BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX8-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
index dcbf45dbe7e62..799f972d76c6a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck %s
-; RUN: llc -global-isel -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX1250 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX1250 %s
; FIXME: Test with SI when argument lowering not broken for f16
; Natural mapping
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
index 12c6f80393106..ccae5c178a7dd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
@@ -7,42 +7,30 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -53,40 +41,28 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__voffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 4095, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll
index 52b488bf42142..4b914bd4a1959 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll
@@ -465,21 +465,15 @@ define amdgpu_ps void @raw_ptr_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK: bb.1 (%ir-block.0):
; CHECK-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; CHECK-NEXT: G_AMDGPU_BUFFER_STORE_SHORT [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; CHECK-NEXT: BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll
index a73704c36ba28..e52daa5c31093 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll
@@ -7,42 +7,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@@ -92,32 +80,22 @@ define amdgpu_ps void @raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
- ; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x i16>)
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
- ; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
- ; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
- ; UNPACKED-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV10]](s32), [[UV11]](s32), [[UV12]](s32), [[UV13]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR2]](<3 x i32>), [[BUILD_VECTOR3]](<4 x s32>), [[C]](i32), [[COPY6]], [[COPY7]], 0, 78, 0, 0 :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE]](<3 x i32>), [[COPY8]](i32), [[REG_SEQUENCE1]](<4 x s32>), [[COPY9]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -189,44 +167,104 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 inreg %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[COPY6]](i32), 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[COPY6]](i32), 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
ret void
@@ -236,44 +274,110 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr6
+ ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr6
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@@ -283,44 +387,112 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(half %val, ptr addrspace(8) %rsrc, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
+ ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]](i32), [[COPY8]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; PACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
+ ; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]](i32), [[COPY8]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@@ -331,42 +503,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret void
@@ -377,42 +537,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret void
@@ -423,42 +571,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret void
@@ -469,42 +605,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
index 68f5295b6c806..5c20e92bd22fe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -8,38 +8,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -121,30 +113,22 @@ define amdgpu_ps void @raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
- ; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
- ; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
- ; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR3]](<3 x i32>), [[BUILD_VECTOR1]](<4 x i32>), [[C]](i32), [[COPY6]], [[COPY7]], 0, 78, 0, 0 :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE1]](<3 x i32>), [[COPY8]](i32), [[REG_SEQUENCE]](<4 x i32>), [[COPY9]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -250,40 +234,96 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -342,40 +382,102 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
@@ -437,40 +539,104 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
@@ -534,38 +700,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
@@ -593,38 +751,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
@@ -652,38 +802,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
@@ -711,38 +853,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
index 1857210367f31..6d84c50b65b08 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@@ -462,36 +462,34 @@ define amdgpu_ps half @struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load.format), [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load.format), [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16), align 1, addrspace 8)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; PACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
index 0d5b69d01a3c5..80c04cc6e4e36 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
@@ -1,12 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
-; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
+; UNSUPPORTED: true
+; RUN: llc -global-isel -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
+; RUN: llc -global-isel -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
+; RUN: llc -global-isel -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
+; RUN: llc -global-isel -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
+; RUN: llc -global-isel -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
+; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
+; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
+; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
; GFX67-LABEL: name: raw_buffer_load_i8_tfe
@@ -337,150 +338,154 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX67: bb.1 (%ir-block.0):
; GFX67-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX67-NEXT: {{ $}}
- ; GFX67-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX67-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX67-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX67-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX67-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX67-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX67-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX67-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX67-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX67-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX67-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX67-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX67-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX67-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX67-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX67-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX67-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX67-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX67-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX67-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX67-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX67-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX67-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX67-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX67-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX67-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX67-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX67-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX67-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX67-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX67-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX67-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX67-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX67-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
+ ; GFX67-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
+ ; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
+ ; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE3]], %subreg.sub2_sub3
+ ; GFX67-NEXT: BUFFER_STORE_SHORT_ADDR64 [[COPY9]], [[REG_SEQUENCE1]], [[REG_SEQUENCE4]], 0, 0, 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+ ; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
+ ; GFX67-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_1]], %subreg.sub0_sub1, [[REG_SEQUENCE5]], %subreg.sub2_sub3
+ ; GFX67-NEXT: BUFFER_STORE_DWORD_ADDR64 [[COPY10]], [[REG_SEQUENCE2]], [[REG_SEQUENCE6]], 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX67-NEXT: S_ENDPGM 0
;
; GFX8-LABEL: name: raw_buffer_load_f16_tfe
; GFX8: bb.1 (%ir-block.0):
; GFX8-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX8-NEXT: {{ $}}
- ; GFX8-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX8-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX8-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX8-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX8-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX8-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX8-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX8-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX8-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX8-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX8-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX8-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX8-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX8-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX8-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX8-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
+ ; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
+ ; GFX8-NEXT: FLAT_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
; GFX910-LABEL: name: raw_buffer_load_f16_tfe
; GFX910: bb.1 (%ir-block.0):
; GFX910-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX910-NEXT: {{ $}}
- ; GFX910-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX910-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX910-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX910-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX910-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX910-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX910-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX910-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX910-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX910-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX910-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX910-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX910-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX910-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX910-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX910-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX910-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX910-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX910-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX910-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX910-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX910-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX910-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX910-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX910-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX910-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX910-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX910-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX910-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX910-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX910-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX910-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX910-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX910-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
+ ; GFX910-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
+ ; GFX910-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
; GFX11-LABEL: name: raw_buffer_load_f16_tfe
; GFX11: bb.1 (%ir-block.0):
; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX11-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX11-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX11-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX11-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX11-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
+ ; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
+ ; GFX11-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
; GFX1200: bb.1 (%ir-block.0):
; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX1200-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX1200-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX1200-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX1200-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1200-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX1200-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX1200-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1200-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX1200-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX1200-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX1200-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1200-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
; GFX1250: bb.1 (%ir-block.0):
; GFX1250-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
- ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
- ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX1250-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
- ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX1250-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX1250-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
- ; GFX1250-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+ ; GFX1250-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { half, i32 } @llvm.amdgcn.struct.buffer.load.sl_f16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue { half, i32 } %res, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
index 72bea2d24f9d7..8f6e5c842f247 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
@@ -1,45 +1,41 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
; UNPACKED-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -129,30 +125,24 @@ define amdgpu_ps void @struct_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
- ; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
- ; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
- ; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR3]](<3 x i32>), [[BUILD_VECTOR1]](<4 x i32>), [[COPY6]](i32), [[COPY7]], [[COPY8]], 0, 0, -1 :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY8]](i32), %subreg.sub0, [[COPY9]](i32), %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE1]](<3 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE]](<4 x i32>), [[COPY10]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -267,59 +257,167 @@ define amdgpu_ps void @struct_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vg
define amdgpu_ps void @struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset(half inreg %val, <4 x i32> %rsrc, i32 inreg %vindex, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; PACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:sgpr(i16) = G_TRUNC [[COPY]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:sgpr(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[BITCAST]](f16)
- ; GFX12-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: bb.2:
+ ; GFX12-NEXT: successors: %bb.3(0x80000000)
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+ ; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: bb.3:
+ ; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: bb.4:
+ ; GFX12-NEXT: successors: %bb.5(0x80000000)
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: bb.5:
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
index 1600dcb2c37d8..92f5bf60e1797 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
@@ -1,4 +1,4 @@
-; RUN: llc -global-isel -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
; RUN: not llc -global-isel < %s -mtriple=amdgpu9.08 2>&1 | FileCheck %s -check-prefix=GFX908
; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(f32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(i32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (f32) on %ir.rsrc.load, align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
index f622f2ecd1f66..e0123072404b3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
@@ -335,36 +335,34 @@ define amdgpu_ps half @struct_ptr_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.ptr.buffer.load.format), [[MV]](p8), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: struct_ptr_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.ptr.buffer.load.format), [[MV]](p8), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; PACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call i16 @llvm.amdgcn.struct.ptr.buffer.load.format.i16(ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
%fval = bitcast i16 %val to half
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll
index 483b003499d08..91e3eb8d8bee4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll
@@ -7,42 +7,34 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_f16__vgpr_val__sgpr_rsrc__
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -96,32 +88,24 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
- ; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x i16>)
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
- ; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
- ; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
- ; UNPACKED-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV10]](s32), [[UV11]](s32), [[UV12]](s32), [[UV13]](s32)
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR2]](<3 x i32>), [[BUILD_VECTOR3]](<4 x s32>), [[COPY6]](i32), [[COPY7]], [[COPY8]], 0, 0, -1 :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY8]](i32), %subreg.sub0, [[COPY9]](i32), %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE]](<3 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE1]](<4 x s32>), [[COPY10]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -198,44 +182,116 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc
define amdgpu_ps void @struct_ptr_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset(half inreg %val, ptr addrspace(8) %rsrc, i32 inreg %vindex, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: struct_ptr_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
+ ; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr4
+ ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY]](i32)
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY6]](i32)
+ ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.2:
+ ; UNPACKED-NEXT: successors: %bb.3(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.3:
+ ; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY9]](i32), %subreg.sub0, [[COPY10]](i32), %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY8]](i32), [[REG_SEQUENCE1]](i64), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.4:
+ ; UNPACKED-NEXT: successors: %bb.5(0x80000000)
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; UNPACKED-NEXT: {{ $}}
+ ; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
+ ; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
- ; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
- ; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
- ; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
- ; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
- ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
- ; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
+ ; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+ ; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+ ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+ ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
+ ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
+ ; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr4
+ ; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+ ; PACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY]](i32)
+ ; PACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
+ ; PACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY6]](i32)
+ ; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.2:
+ ; PACKED-NEXT: successors: %bb.3(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
+ ; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
+ ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
+ ; PACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
+ ; PACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
+ ; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
+ ; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
+ ; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
+ ; PACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
+ ; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.3:
+ ; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY9]](i32), %subreg.sub0, [[COPY10]](i32), %subreg.sub1
+ ; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY8]](i32), [[REG_SEQUENCE1]](i64), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
+ ; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.4:
+ ; PACKED-NEXT: successors: %bb.5(0x80000000)
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
+ ; PACKED-NEXT: {{ $}}
+ ; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll
index ea6725f6a36d8..ec1bf960b56f2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll
@@ -5,16 +5,35 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
define i16 @v_powi_f16(i16 %l, i32 %r) {
-; GFX78-LABEL: v_powi_f16:
-; GFX78: ; %bb.0:
-; GFX78-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX78-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX78-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX78-NEXT: v_log_f32_e32 v0, v0
-; GFX78-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX78-NEXT: v_exp_f32_e32 v0, v0
-; GFX78-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX78-NEXT: s_setpc_b64 s[30:31]
+; GFX7-LABEL: v_powi_f16:
+; GFX7: ; %bb.0:
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX7-NEXT: v_mov_b32_e32 v2, 0xc2fc0000
+; GFX7-NEXT: v_mov_b32_e32 v3, 0x42800000
+; GFX7-NEXT: v_log_f32_e32 v0, v0
+; GFX7-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX7-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; GFX7-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX7-NEXT: v_exp_f32_e32 v0, v0
+; GFX7-NEXT: v_not_b32_e32 v1, 63
+; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT: v_ldexp_f32_e32 v0, v0, v1
+; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_powi_f16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX8-NEXT: v_log_f32_e32 v0, v0
+; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
+; GFX8-NEXT: v_exp_f32_e32 v0, v0
+; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_powi_f16:
; GFX11-TRUE16: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index c79009e14e669..a7658cea07b1b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -710,6 +710,7 @@ define amdgpu_ps i16 @s_lshr_i16_15(i16 inreg %value) {
define amdgpu_ps half @lshr_i16_sv(i16 inreg %value, i16 %amount) {
; GFX6-LABEL: lshr_i16_sv:
; GFX6: ; %bb.0:
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_lshr_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
@@ -736,6 +737,7 @@ define amdgpu_ps half @lshr_i16_sv(i16 inreg %value, i16 %amount) {
define amdgpu_ps half @lshr_i16_vs(i16 %value, i16 inreg %amount) {
; GFX6-LABEL: lshr_i16_vs:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/madmix-constant-bus-violation.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/madmix-constant-bus-violation.ll
index d05d01fdc9431..480abafb0906c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/madmix-constant-bus-violation.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/madmix-constant-bus-violation.ll
@@ -1,13 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 %s -o - | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 %s -o - | FileCheck %s
define float @test_fmamix_constant_bus_violation_sss(i32 inreg %val.0, i32 inreg %val.1, i32 inreg %val.2) #0 {
; CHECK-LABEL: test_fmamix_constant_bus_violation_sss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v0, s17
-; CHECK-NEXT: v_mov_b32_e32 v1, s18
-; CHECK-NEXT: v_mad_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s6, s18, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v0, s5
+; CHECK-NEXT: v_mov_b32_e32 v1, s6
+; CHECK-NEXT: v_mad_mix_f32 v0, s4, v0, v1 op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@@ -29,8 +32,10 @@ define float @test_fmamix_constant_bus_violation_ssv(i32 inreg %val.0, i32 inreg
; CHECK-LABEL: test_fmamix_constant_bus_violation_ssv:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
-; CHECK-NEXT: v_mad_mix_f32 v0, s16, v1, v0 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_mad_mix_f32 v0, s4, v1, v0 op_sel:[0,0,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@@ -52,8 +57,10 @@ define float @test_fmamix_constant_bus_violation_svs(i32 inreg %val.0, i32 %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_svs:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
-; CHECK-NEXT: v_mad_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_mad_mix_f32 v0, s4, v0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@@ -75,8 +82,10 @@ define float @test_fmamix_constant_bus_violation_vss(i32 %val.0, i32 inreg %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_vss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
-; CHECK-NEXT: v_mad_mix_f32 v0, v0, s16, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; CHECK-NEXT: s_lshr_b32 s5, s17, 16
+; CHECK-NEXT: s_lshr_b32 s4, s16, 16
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_mad_mix_f32 v0, v0, s4, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 2594d49f1b5e6..c0487df72569d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -2597,23 +2597,28 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: saddsat_i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_min_i32 s2, s0, 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_max_i32 s1, s0, 0
+; GFX6-NEXT: s_sub_i32 s2, 0x80000000, s2
+; GFX6-NEXT: s_sub_i32 s1, 0x7fffffff, s1
+; GFX6-NEXT: v_max_i32_e32 v0, s2, v0
+; GFX6-NEXT: v_min_i32_e32 v0, s1, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: s_movk_i32 s0, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i16_sv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v0
+; GFX8-NEXT: s_sext_i32_i16 s1, s0
+; GFX8-NEXT: s_max_i32 s2, s1, 0
+; GFX8-NEXT: s_min_i32 s1, s1, 0
+; GFX8-NEXT: s_sub_i32 s1, 0x8000, s1
+; GFX8-NEXT: s_sub_i32 s2, 0x7fff, s2
+; GFX8-NEXT: v_max_i16_e32 v0, s1, v0
+; GFX8-NEXT: v_min_i16_e32 v0, s2, v0
; GFX8-NEXT: v_add_u16_e32 v0, s0, v0
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], s0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i16_sv:
@@ -2643,25 +2648,27 @@ define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @saddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: saddsat_i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: s_movk_i32 s0, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_i32_e32 v2, 0, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_max_i32_e32 v1, 0, v0
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 0x80000000, v2
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, 0x7fffffff, v1
+; GFX6-NEXT: v_max_i32_e32 v2, s0, v2
+; GFX6-NEXT: v_min_i32_e32 v1, v2, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i16_vs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_add_u16_e32 v1, s0, v0
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s0, 0
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v1, v0
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, 15, v1
-; GFX8-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
-; GFX8-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT: v_min_i16_e32 v2, 0, v0
+; GFX8-NEXT: v_max_i16_e32 v1, 0, v0
+; GFX8-NEXT: v_sub_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT: v_sub_u16_e32 v1, 0x7fff, v1
+; GFX8-NEXT: v_max_i16_e32 v2, s0, v2
+; GFX8-NEXT: v_min_i16_e32 v1, v2, v1
+; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
index c6210e6c919db..79e61d7f17326 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
@@ -5,8 +5,8 @@ define half @test_s16(half %a) #0 {
; GCN-LABEL: test_s16:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt half %a, 0.0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
index 569ec2be38f8e..ff4d6f7cd6640 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
@@ -3,7 +3,7 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; Test optimization to reduce shifts to narrower sizes.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
index 50d94aa295aa6..a5fbd0a99d627 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
@@ -650,6 +650,7 @@ define amdgpu_ps i16 @s_shl_i16_15(i16 inreg %value) {
define amdgpu_ps half @shl_i16_sv(i16 inreg %value, i16 %amount) {
; GFX6-LABEL: shl_i16_sv:
; GFX6: ; %bb.0:
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshl_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
;
@@ -675,6 +676,7 @@ define amdgpu_ps half @shl_i16_sv(i16 inreg %value, i16 %amount) {
define amdgpu_ps half @shl_i16_vs(i16 %value, i16 inreg %amount) {
; GFX6-LABEL: shl_i16_vs:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_lshlrev_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
index ed1b4e9cc325e..24c48ed549d6b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
-; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
define i32 @test_min_max_ValK0_K1_i32(i32 %a) {
; GFX89-LABEL: test_min_max_ValK0_K1_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index e05a8d60f64bc..fbd35842fc6c9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -2601,23 +2601,28 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: ssubsat_i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_max_i32 s1, s0, -1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s1, s1, 0x80000001
+; GFX6-NEXT: s_min_i32 s2, s0, -1
+; GFX6-NEXT: s_add_i32 s2, s2, 0x80000000
+; GFX6-NEXT: v_max_i32_e32 v0, s1, v0
+; GFX6-NEXT: v_min_i32_e32 v0, s2, v0
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: s_movk_i32 s0, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i16_sv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, s0, v0
+; GFX8-NEXT: s_sext_i32_i16 s1, s0
+; GFX8-NEXT: s_max_i32 s2, s1, -1
+; GFX8-NEXT: s_add_i32 s2, s2, 0x8001
+; GFX8-NEXT: s_min_i32 s1, s1, -1
+; GFX8-NEXT: s_add_i32 s1, s1, 0x8000
+; GFX8-NEXT: v_max_i16_e32 v0, s2, v0
+; GFX8-NEXT: v_min_i16_e32 v0, s1, v0
; GFX8-NEXT: v_sub_u16_e32 v0, s0, v0
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i16_sv:
@@ -2647,23 +2652,27 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: ssubsat_i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: s_movk_i32 s0, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v1, -1, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000001, v1
+; GFX6-NEXT: v_min_i32_e32 v2, -1, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0x80000000, v2
+; GFX6-NEXT: v_max_i32_e32 v1, s0, v1
+; GFX6-NEXT: v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i16_vs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, s0, v0
-; GFX8-NEXT: v_subrev_u16_e32 v0, s0, v0
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT: v_max_i16_e32 v1, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v1, 0x8001, v1
+; GFX8-NEXT: v_min_i16_e32 v2, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT: v_max_i16_e32 v1, s0, v1
+; GFX8-NEXT: v_min_i16_e32 v1, v1, v2
+; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index a702ade4dadbe..3b6106b915e03 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -5,7 +5,7 @@
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal < %s | FileCheck --check-prefix=GFX11 %s
; FIXME:
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
+; XUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
define amdgpu_kernel void @store_lds_v3i32(ptr addrspace(3) %out, <3 x i32> %x) {
; GFX9-LABEL: store_lds_v3i32:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
index 58329bba79ec2..21afa81ba8c2d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
@@ -41,7 +41,7 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b16_e32 v2.l, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_fmac_f16_e64 v2.l, s0, s1
+; GFX11-NEXT: v_fma_f16 v2.l, s0, s1, v2.l
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -83,10 +83,10 @@ define void @v_constained_fma_f16_fpexcept_strict_div(half %x, half %y, half %z,
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX942-NEXT: global_store_short v[4:5], v0, off
+; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -463,9 +463,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX8-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s17
-; GFX8-NEXT: v_mov_b32_e32 v3, s18
-; GFX8-NEXT: v_fma_f16 v2, s16, v2, -v3
+; GFX8-NEXT: v_mov_b32_e32 v2, s16
+; GFX8-NEXT: v_mov_b32_e32 v3, s17
+; GFX8-NEXT: v_fma_f16 v2, v2, v3, -s18
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -473,9 +473,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_fma_f16 v2, s16, v2, -v3
+; GFX900-NEXT: v_mov_b32_e32 v2, s16
+; GFX900-NEXT: v_mov_b32_e32 v3, s17
+; GFX900-NEXT: v_fma_f16 v2, v2, v3, -s18
; GFX900-NEXT: global_store_short v[0:1], v2, off
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
@@ -483,9 +483,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_fma_f16 v2, s0, v2, -v3
+; GFX942-NEXT: v_mov_b32_e32 v2, s0
+; GFX942-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-NEXT: v_fma_f16 v2, v2, v3, -s2
; GFX942-NEXT: global_store_short v[0:1], v2, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -493,9 +493,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f16 v2.l, s0, s1, -v2.l
+; GFX11-NEXT: v_fma_f16 v2.l, s0, v2.l, -s2
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -540,10 +540,10 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_div(half %x, half %y, hal
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, v0, v1, -v2
-; GFX942-NEXT: global_store_short v[4:5], v0, off
+; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -653,10 +653,10 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_div(half %x, half %y
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, -v0, -v1, v2
-; GFX942-NEXT: global_store_short v[4:5], v0, off
+; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@@ -767,10 +767,10 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_div(half %x, half %y
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
-; GFX942-NEXT: global_store_short v[4:5], v0, off
+; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll
index 335fe8066c000..de4d43ee67070 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
define i16 @s_sub_i16(i16 inreg %a, i16 inreg %b) {
; GFX7-LABEL: s_sub_i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 9f9d47c09bdfb..597309977ebfa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -1811,10 +1811,12 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: uaddsat_i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_not_b32 s1, s0
+; GFX6-NEXT: v_min_u32_e32 v0, s1, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_i16_sv:
@@ -1849,10 +1851,12 @@ define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @uaddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: uaddsat_i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_not_b32_e32 v1, v0
+; GFX6-NEXT: v_min_u32_e32 v1, s0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index 2a34ead5e072f..7108578ca55e5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -1699,10 +1699,11 @@ define amdgpu_ps i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @usubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: usubsat_i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_max_u32_e32 v1, s0, v0
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v1, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_u32_e32 v0, s0, v0
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_i16_sv:
@@ -1737,10 +1738,11 @@ define amdgpu_ps half @usubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @usubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: usubsat_i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_max_u32_e32 v0, s0, v0
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_min_u32_e32 v1, s0, v0
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/add-max.ll b/llvm/test/CodeGen/AMDGPU/add-max.ll
index d7cc8cc375873..52fda87f3c33f 100644
--- a/llvm/test/CodeGen/AMDGPU/add-max.ll
+++ b/llvm/test/CodeGen/AMDGPU/add-max.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GCN,SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GISEL %s
define amdgpu_ps float @add_max_u32_vvv(i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: add_max_u32_vvv:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-sin-cos-f16-f32.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-sin-cos-f16-f32.ll
index 83b3effdb77d0..aea9a6564096a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn-sin-cos-f16-f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-sin-cos-f16-f32.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "s_setreg_imm32_b32" --filter-out "shader" --version 6
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
define amdgpu_ps float @v_sin_f32(float %src) #1 {
; GCN-LABEL: v_sin_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
index 04c78cd3cb0d1..eb9cc4a7b524d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX9-GISEL
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX10-GISEL
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX11-GISEL
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX12-GISEL
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX9-GISEL
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX10-GISEL
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX11-GISEL
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX12-GISEL
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX9-SDAG
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX10-SDAG
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck %s -check-prefix=GFX11-SDAG
@@ -127,11 +127,8 @@ define i32 @global_atomic_usub_sat_offset(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, 0x1000, v0
-; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], vcc
-; GFX9-GISEL-NEXT: v_addc_co_u32_e64 v4, s[4:5], 0, v1, s[4:5]
-; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-GISEL-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc
+; GFX9-GISEL-NEXT: global_load_dword v0, v[3:4], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -371,23 +368,20 @@ define void @global_atomic_usub_sat_offset_nortn(ptr addrspace(1) %ptr, i32 %dat
; GFX9-GISEL-LABEL: global_atomic_usub_sat_offset_nortn:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, 0x1000, v0
-; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], vcc
-; GFX9-GISEL-NEXT: v_addc_co_u32_e64 v4, s[4:5], 0, v1, s[4:5]
+; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0
; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-GISEL-NEXT: global_load_dword v1, v[0:1], off
+; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_sub_u32_e64 v0, v1, v2 clamp
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v[3:4], v[0:1], off glc
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -503,33 +497,31 @@ define void @global_atomic_usub_sat_offset_nortn(ptr addrspace(1) %ptr, i32 %dat
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace(1) %ptr, i32 %data, ptr addrspace(1) %dst) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[2:3], 0x1000
-; GFX9-GISEL-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-GISEL-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-GISEL-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-GISEL-NEXT: v_sub_u32_e64 v2, v3, s4 clamp
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[2:3] glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v1, v2, s4 clamp
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
-; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
@@ -663,26 +655,24 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[2:3], 0x1000
-; GFX9-GISEL-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-GISEL-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-GISEL-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_sub_u32_e64 v0, v1, s4 clamp
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_endpgm
@@ -797,17 +787,17 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xffff0000
; GFX9-GISEL-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v6, v2 clamp
+; GFX9-GISEL-NEXT: v_and_or_b32 v5, v6, v4, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB6_1
@@ -868,7 +858,7 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_16:
@@ -901,7 +891,7 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_16:
@@ -1025,17 +1015,17 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:2048
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xffff0000
; GFX9-GISEL-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2048 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v6, v2 clamp
+; GFX9-GISEL-NEXT: v_and_or_b32 v5, v6, v4, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB7_1
@@ -1097,7 +1087,7 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_offset_16:
@@ -1130,7 +1120,7 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_offset_16:
@@ -1256,12 +1246,12 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffff0000
; GFX9-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v5, v3
; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
@@ -1472,12 +1462,12 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off offset:2048
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s6, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffff0000
; GFX9-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v4, v2 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v5, v3
; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
@@ -1693,30 +1683,30 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff0000
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x800
+; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x800
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
; GFX9-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v3, s4 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, s5, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:2048 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v3, s4 clamp
+; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v0, v2
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v1, v[2:3], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_store_short v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_short v0, v2, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
@@ -1724,11 +1714,9 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0x800
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
-; GFX10-GISEL-NEXT: s_add_u32 s0, s0, 0x800
-; GFX10-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX10-GISEL-NEXT: s_mov_b32 s3, 0
@@ -1759,11 +1747,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s3
; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
; GFX11-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1793,11 +1780,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1971,17 +1957,17 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0xffff0000
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x800
+; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x800
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v1, s4 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, s5, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:2048 glc
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v2, v0
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1997,11 +1983,9 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0x800
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
-; GFX10-GISEL-NEXT: s_add_u32 s0, s0, 0x800
-; GFX10-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX10-GISEL-NEXT: s_mov_b32 s3, 0
@@ -2027,11 +2011,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
; GFX11-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2056,11 +2039,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2209,17 +2191,21 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffffff00
; GFX9-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v7
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, v2 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v6, v7, v5, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB12_1
@@ -2232,22 +2218,23 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
; GFX10-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v3, 8, v6
; GFX10-GISEL-NEXT: v_sub_nc_u16 v3, v3, v2 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v5, 0xffffff00, v6, v3
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
-; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB12_1
@@ -2260,17 +2247,20 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -2284,7 +2274,7 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_8:
@@ -2295,17 +2285,19 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v3, v[0:1], off
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
; GFX12-GISEL-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2319,7 +2311,7 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_8:
@@ -2449,17 +2441,21 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffffff00
; GFX9-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:1024 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v7
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, v2 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v6, v7, v5, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB13_1
@@ -2472,22 +2468,23 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
; GFX10-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v3, 8, v6
; GFX10-GISEL-NEXT: v_sub_nc_u16 v3, v3, v2 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v5, 0xffffff00, v6, v3
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:1024 glc
+; GFX10-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:1024 glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
-; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB13_1
@@ -2500,17 +2497,20 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v3, v[0:1], off offset:1024
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 glc
@@ -2524,7 +2524,7 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_offset_8:
@@ -2535,17 +2535,19 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v3, v[0:1], off offset:1024
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
; GFX12-GISEL-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2559,7 +2561,7 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_offset_8:
@@ -2688,20 +2690,24 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_nortn_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off
+; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0xffffff00
; GFX9-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v2, v4 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v6, v2
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB14_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2712,14 +2718,15 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v4, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
; GFX10-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v3
; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, v4 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v2, 0xffffff00, v3, v2
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
@@ -2739,16 +2746,18 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -2773,16 +2782,17 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
; GFX12-GISEL-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2920,20 +2930,24 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX9-GISEL-LABEL: global_atomic_usub_sat_offset_nortn_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v4, v[0:1], off offset:1024
+; GFX9-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s6, 0xff00
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0xffffff00
; GFX9-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v3, v4, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:1024 glc
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v2, v4 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v6, v2
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB15_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2944,14 +2958,15 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: global_load_dword v3, v[0:1], off offset:1024
-; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v4, 8, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
; GFX10-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v3
; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, v4 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v2, 0xffffff00, v3, v2
; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-GISEL-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:1024 glc
@@ -2971,16 +2986,18 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v4, v[0:1], off offset:1024
-; GFX11-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 glc
@@ -3005,16 +3022,17 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: global_load_b32 v4, v[0:1], off offset:1024
-; GFX12-GISEL-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
; GFX12-GISEL-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v2.h, 8, v4.l
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v3.l, 8, v2.h
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -3153,33 +3171,36 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s5, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffffff00
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x400
-; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x400
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s5
; GFX9-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v0, v3, s5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, s4, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:1024 glc
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, s4 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v1, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v2, v[3:4], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_byte v0, v3, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
@@ -3188,24 +3209,25 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX10-GISEL-NEXT: s_mov_b32 s2, 0
; GFX10-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX10-GISEL-NEXT: v_sub_nc_u16 v1, v1, s3 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
-; GFX10-GISEL-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] offset:1024 glc
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v3
+; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, s3 clamp
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v2, 0xffffff00, v3, v2
+; GFX10-GISEL-NEXT: global_atomic_cmpswap v2, v0, v[2:3], s[0:1] offset:1024 glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
-; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX10-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB16_1
@@ -3214,7 +3236,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_byte v0, v2, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
@@ -3222,22 +3244,24 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s3, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s2, s[0:1], 0x400
-; GFX11-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX11-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-GISEL-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v1.l, 8, v2.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_nc_u16 v1.l, v1.l, s3 clamp
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v1, v1.l
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:1024 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3259,10 +3283,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x400
-; GFX12-GISEL-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
@@ -3270,11 +3294,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-GISEL-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v1.l, 8, v2.l
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_sub_nc_u16 v1.l, v1.l, s2 clamp
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v1, v1.l
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX12-GISEL-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -3445,20 +3470,23 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s5, s[8:9], 0x8
+; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffffff00
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s6, s[0:1], 0x400
-; GFX9-GISEL-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x400
+; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_sub_u16_sdwa v0, v1, s5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, s4, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
+; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v0, 8, v1
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v0, s4 clamp
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v3, v0
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v4, v[0:1], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -3475,17 +3503,18 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xff
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX10-GISEL-NEXT: s_mov_b32 s2, 0
; GFX10-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b16 v0, 8, v1
; GFX10-GISEL-NEXT: v_sub_nc_u16 v0, v0, s3 clamp
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX10-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3504,20 +3533,23 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_load_b32 s3, s[4:5], 0x8
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s2, s[0:1], 0x400
-; GFX11-GISEL-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX11-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX11-GISEL-NEXT: v_lshlrev_b16 v0.l, 8, v1.l
; GFX11-GISEL-NEXT: v_sub_nc_u16 v0.l, v0.l, s3 clamp
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-GISEL-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:1024 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -3535,20 +3567,22 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x400
-; GFX12-GISEL-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX12-GISEL-NEXT: v_lshlrev_b16 v0.l, 8, v1.l
; GFX12-GISEL-NEXT: v_sub_nc_u16 v0.l, v0.l, s2 clamp
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX12-GISEL-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/bfi_int.ll b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
index a19bc6529e6ea..5bde70553ea9f 100644
--- a/llvm/test/CodeGen/AMDGPU/bfi_int.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck -enable-var-scope -check-prefix=GFX7 %s
; RUN: llc -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8 %s
; RUN: llc -mtriple=amdgpu10.31 < %s | FileCheck -enable-var-scope -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-GISEL %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.31 < %s | FileCheck -enable-var-scope -check-prefix=GFX10-GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu10.31 < %s | FileCheck -enable-var-scope -check-prefix=GFX10-GISEL %s
; BFI_INT Definition pattern from ISA docs
; (y & x) | (z & ~x)
diff --git a/llvm/test/CodeGen/AMDGPU/bitop3.ll b/llvm/test/CodeGen/AMDGPU/bitop3.ll
index f3870bea2ba84..d87e423ba2ea8 100644
--- a/llvm/test/CodeGen/AMDGPU/bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitop3.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.50-- < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-SDAG %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.50-- < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu9.50-- < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-SDAG-FAKE16,GFX1250-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-SDAG-TRUE16,GFX1250-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50-- -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16,GFX1250-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.50-- -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16,GFX1250-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.50-- -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-TRUE16,GFX1250-TRUE16 %s
; ========= Single bit functions =========
@@ -718,10 +718,16 @@ define amdgpu_ps half @not_and_and_not_and_b16(i16 %a, i16 %b, i16 %c) {
}
define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
-; GFX950-LABEL: test_xor3_b16:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0x96
-; GFX950-NEXT: ; return to shader part epilog
+; GFX950-SDAG-LABEL: test_xor3_b16:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0x96
+; GFX950-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: test_xor3_b16:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_xor3_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@@ -739,21 +745,13 @@ define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0x96
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: test_xor3_b16:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0x96
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-GISEL-TRUE16-LABEL: test_xor3_b16:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: v_xor3_b32 v0, v0, v1, v2
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: test_xor3_b16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_xor3_b32 v0, v0, v1, v2
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%xor1 = xor i16 %a, %b
%xor2 = xor i16 %xor1, %c
%ret_cast = bitcast i16 %xor2 to half
@@ -761,10 +759,15 @@ define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
}
define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
-; GFX950-LABEL: test_or3_b16:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xfe
-; GFX950-NEXT: ; return to shader part epilog
+; GFX950-SDAG-LABEL: test_or3_b16:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xfe
+; GFX950-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: test_or3_b16:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_or3_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@@ -782,21 +785,13 @@ define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xfe
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: test_or3_b16:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xfe
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-GISEL-TRUE16-LABEL: test_or3_b16:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: v_or3_b32 v0, v0, v1, v2
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: test_or3_b16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%or1 = or i16 %a, %b
%or2 = or i16 %or1, %c
%ret_cast = bitcast i16 %or2 to half
@@ -804,10 +799,15 @@ define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
}
define amdgpu_ps half @test_and_or_b16(i16 %a, i16 %b, i16 %c) {
-; GFX950-LABEL: test_and_or_b16:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xec
-; GFX950-NEXT: ; return to shader part epilog
+; GFX950-SDAG-LABEL: test_and_or_b16:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xec
+; GFX950-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: test_and_or_b16:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
+; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_and_or_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@@ -825,21 +825,13 @@ define amdgpu_ps half @test_and_or_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xec
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: test_and_or_b16:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xec
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-GISEL-TRUE16-LABEL: test_and_or_b16:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, v0, v1, v2
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: test_and_or_b16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%and1 = and i16 %a, %b
%or1 = or i16 %and1, %c
%ret_cast = bitcast i16 %or1 to half
diff --git a/llvm/test/CodeGen/AMDGPU/bitreverse.ll b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
index bf001c0605a82..b0adf4a58d85a 100644
--- a/llvm/test/CodeGen/AMDGPU/bitreverse.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
@@ -2,11 +2,11 @@
; RUN: llc < %s -mtriple=amdgpu6.00-- | FileCheck %s --check-prefix=SI
; RUN: llc < %s -mtriple=amdgpu8.02-- -mattr=-flat-for-global | FileCheck %s --check-prefix=FLAT
; RUN: llc < %s -mtriple=amdgpu8.03-- -mattr=-flat-for-global | FileCheck %s --check-prefix=FLAT
-; RUN: llc < %s -mtriple=amdgpu8.03-- -global-isel -global-isel-abort=2 | FileCheck %s --check-prefix=GISEL
+; RUN: llc < %s -mtriple=amdgpu8.03-- -global-isel | FileCheck %s --check-prefix=GISEL
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=+real-true16 -mattr=-flat-for-global | FileCheck %s --check-prefixes=GFX11-FLAT,GFX11-FLAT-TRUE16
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=-real-true16 -mattr=-flat-for-global | FileCheck %s --check-prefixes=GFX11-FLAT,GFX11-FLAT-FAKE16
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=+real-true16 -global-isel | FileCheck %s --check-prefixes=GFX11-GISEL,GFX11-GISEL-TRUE16
-; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=-real-true16 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes=GFX11-GISEL,GFX11-GISEL-FAKE16
+; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=-real-true16 -global-isel | FileCheck %s --check-prefixes=GFX11-GISEL,GFX11-GISEL-FAKE16
declare i32 @llvm.amdgcn.workitem.id.x() #1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
index 816608774bfd2..61ff755f99182 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=SDAG %s
define amdgpu_kernel void @buffer_ptr_vector_ops(ptr addrspace(1) %somewhere) {
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
index 1bfb37d01fcb1..444d74166d307 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
@@ -2,9 +2,9 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16,GFX11-FAKE16-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16,GFX11-FAKE16-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16,GFX11-FAKE16-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-GISEL %s
@@ -81,23 +81,45 @@ define void @undef_lo_v2f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: undef_lo_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: ;;#ASMSTART
-; GFX9-NEXT: ; use v0
-; GFX9-NEXT: ;;#ASMEND
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: undef_lo_v2f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: ;;#ASMSTART
+; GFX9-SDAG-NEXT: ; use v0
+; GFX9-SDAG-NEXT: ;;#ASMEND
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: undef_lo_v2f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: ;;#ASMSTART
-; GFX11-FAKE16-NEXT: ; use v0
-; GFX11-FAKE16-NEXT: ;;#ASMEND
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: undef_lo_v2f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT: ;;#ASMSTART
+; GFX9-GISEL-NEXT: ; use v0
+; GFX9-GISEL-NEXT: ;;#ASMEND
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-SDAG-LABEL: undef_lo_v2f16:
+; GFX11-FAKE16-SDAG: ; %bb.0:
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-SDAG-NEXT: ; use v0
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
+; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-GISEL-LABEL: undef_lo_v2f16:
+; GFX11-FAKE16-GISEL: ; %bb.0:
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e64 v1, 0xffff, s0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-GISEL-NEXT: ; use v0
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
+; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_lo_v2f16:
; GFX11-TRUE16: ; %bb.0:
@@ -124,26 +146,50 @@ define void @undef_lo_op_v2f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: undef_lo_op_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
-; GFX9-NEXT: ;;#ASMSTART
-; GFX9-NEXT: ; use v0
-; GFX9-NEXT: ;;#ASMEND
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: undef_lo_op_v2f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX9-SDAG-NEXT: ;;#ASMSTART
+; GFX9-SDAG-NEXT: ; use v0
+; GFX9-SDAG-NEXT: ;;#ASMEND
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: undef_lo_op_v2f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
-; GFX11-FAKE16-NEXT: ;;#ASMSTART
-; GFX11-FAKE16-NEXT: ; use v0
-; GFX11-FAKE16-NEXT: ;;#ASMEND
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: undef_lo_op_v2f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX9-GISEL-NEXT: ;;#ASMSTART
+; GFX9-GISEL-NEXT: ; use v0
+; GFX9-GISEL-NEXT: ;;#ASMEND
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-SDAG-LABEL: undef_lo_op_v2f16:
+; GFX11-FAKE16-SDAG: ; %bb.0:
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-SDAG-NEXT: ; use v0
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
+; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-GISEL-LABEL: undef_lo_op_v2f16:
+; GFX11-FAKE16-GISEL: ; %bb.0:
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e64 v1, 0xffff, s0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX11-FAKE16-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-GISEL-NEXT: ; use v0
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
+; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_lo_op_v2f16:
; GFX11-TRUE16: ; %bb.0:
@@ -319,23 +365,45 @@ define void @undef_lo3_v4f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: undef_lo3_v4f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: ;;#ASMSTART
-; GFX9-NEXT: ; use v[0:1]
-; GFX9-NEXT: ;;#ASMEND
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: undef_lo3_v4f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: ;;#ASMSTART
+; GFX9-SDAG-NEXT: ; use v[0:1]
+; GFX9-SDAG-NEXT: ;;#ASMEND
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: undef_lo3_v4f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: ;;#ASMSTART
-; GFX11-FAKE16-NEXT: ; use v[0:1]
-; GFX11-FAKE16-NEXT: ;;#ASMEND
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: undef_lo3_v4f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, s4, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT: ;;#ASMSTART
+; GFX9-GISEL-NEXT: ; use v[0:1]
+; GFX9-GISEL-NEXT: ;;#ASMEND
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-SDAG-LABEL: undef_lo3_v4f16:
+; GFX11-FAKE16-SDAG: ; %bb.0:
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-SDAG-NEXT: ; use v[0:1]
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
+; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-GISEL-LABEL: undef_lo3_v4f16:
+; GFX11-FAKE16-GISEL: ; %bb.0:
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e64 v1, 0xffff, s0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-GISEL-NEXT: ; use v[0:1]
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
+; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_lo3_v4f16:
; GFX11-TRUE16: ; %bb.0:
@@ -504,29 +572,67 @@ define void @undef_hi_v2i16(i16 %arg0) {
}
define void @undef_hi_v2f16(half %arg0) {
-; GFX8-LABEL: undef_hi_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v0
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_hi_v2f16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v0
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: undef_hi_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: ;;#ASMSTART
-; GFX9-NEXT: ; use v0
-; GFX9-NEXT: ;;#ASMEND
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: undef_hi_v2f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v0
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: undef_hi_v2f16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: ;;#ASMSTART
-; GFX11-NEXT: ; use v0
-; GFX11-NEXT: ;;#ASMEND
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: undef_hi_v2f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: ;;#ASMSTART
+; GFX9-SDAG-NEXT: ; use v0
+; GFX9-SDAG-NEXT: ;;#ASMEND
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: undef_hi_v2f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: ;;#ASMSTART
+; GFX9-GISEL-NEXT: ; use v0
+; GFX9-GISEL-NEXT: ;;#ASMEND
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-SDAG-LABEL: undef_hi_v2f16:
+; GFX11-FAKE16-SDAG: ; %bb.0:
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-SDAG-NEXT: ; use v0
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
+; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-GISEL-LABEL: undef_hi_v2f16:
+; GFX11-FAKE16-GISEL: ; %bb.0:
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-GISEL-NEXT: ; use v0
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
+; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: undef_hi_v2f16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: ;;#ASMSTART
+; GFX11-TRUE16-NEXT: ; use v0
+; GFX11-TRUE16-NEXT: ;;#ASMEND
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
%undef.hi = insertelement <2 x half> poison, half %arg0, i32 0
call void asm sideeffect "; use $0", "v"(<2 x half> %undef.hi);
ret void
@@ -543,23 +649,46 @@ define void @undef_hi_op_v2f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: undef_hi_op_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
-; GFX9-NEXT: ;;#ASMSTART
-; GFX9-NEXT: ; use v0
-; GFX9-NEXT: ;;#ASMEND
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: undef_hi_op_v2f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX9-SDAG-NEXT: ;;#ASMSTART
+; GFX9-SDAG-NEXT: ; use v0
+; GFX9-SDAG-NEXT: ;;#ASMEND
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: undef_hi_op_v2f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
-; GFX11-FAKE16-NEXT: ;;#ASMSTART
-; GFX11-FAKE16-NEXT: ; use v0
-; GFX11-FAKE16-NEXT: ;;#ASMEND
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: undef_hi_op_v2f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX9-GISEL-NEXT: ;;#ASMSTART
+; GFX9-GISEL-NEXT: ; use v0
+; GFX9-GISEL-NEXT: ;;#ASMEND
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-SDAG-LABEL: undef_hi_op_v2f16:
+; GFX11-FAKE16-SDAG: ; %bb.0:
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-SDAG-NEXT: ; use v0
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
+; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-GISEL-LABEL: undef_hi_op_v2f16:
+; GFX11-FAKE16-GISEL: ; %bb.0:
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX11-FAKE16-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-GISEL-NEXT: ; use v0
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
+; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-SDAG-LABEL: undef_hi_op_v2f16:
; GFX11-TRUE16-SDAG: ; %bb.0:
@@ -730,29 +859,68 @@ define void @undef_hi3_v4i16(i16 %arg0) {
}
define void @undef_hi3_v4f16(half %arg0) {
-; GFX8-LABEL: undef_hi3_v4f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v[0:1]
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_hi3_v4f16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v[0:1]
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: undef_hi3_v4f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: ;;#ASMSTART
-; GFX9-NEXT: ; use v[0:1]
-; GFX9-NEXT: ;;#ASMEND
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: undef_hi3_v4f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v[0:1]
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: undef_hi3_v4f16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: ;;#ASMSTART
-; GFX11-NEXT: ; use v[0:1]
-; GFX11-NEXT: ;;#ASMEND
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: undef_hi3_v4f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: ;;#ASMSTART
+; GFX9-SDAG-NEXT: ; use v[0:1]
+; GFX9-SDAG-NEXT: ;;#ASMEND
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: undef_hi3_v4f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: ;;#ASMSTART
+; GFX9-GISEL-NEXT: ; use v[0:1]
+; GFX9-GISEL-NEXT: ;;#ASMEND
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-SDAG-LABEL: undef_hi3_v4f16:
+; GFX11-FAKE16-SDAG: ; %bb.0:
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-SDAG-NEXT: ; use v[0:1]
+; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
+; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-GISEL-LABEL: undef_hi3_v4f16:
+; GFX11-FAKE16-GISEL: ; %bb.0:
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
+; GFX11-FAKE16-GISEL-NEXT: ; use v[0:1]
+; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
+; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: undef_hi3_v4f16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: ;;#ASMSTART
+; GFX11-TRUE16-NEXT: ; use v[0:1]
+; GFX11-TRUE16-NEXT: ;;#ASMEND
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
%undef.hi = insertelement <4 x half> poison, half %arg0, i32 0
call void asm sideeffect "; use $0", "v"(<4 x half> %undef.hi);
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
index aac43267b2785..43e3071662ba3 100644
--- a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GISEL %s
declare hidden void @external_void_func_i8_inreg(i8 inreg) #0
declare hidden void @external_void_func_i16_inreg(i32 inreg) #0
diff --git a/llvm/test/CodeGen/AMDGPU/call-return-types.ll b/llvm/test/CodeGen/AMDGPU/call-return-types.ll
index a4765fe7df0c9..c92fda1787758 100644
--- a/llvm/test/CodeGen/AMDGPU/call-return-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-return-types.ll
@@ -2,9 +2,9 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu7.01-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GFX7 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX11 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu8.03-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu7.01-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; Ideally, we would also like to test GlobalISel with gfx11 but we are currently blocked on llvm-project#166501.
diff --git a/llvm/test/CodeGen/AMDGPU/commute-compares-scalar-float.ll b/llvm/test/CodeGen/AMDGPU/commute-compares-scalar-float.ll
index 6d8e78acdfc00..6c654c8be097e 100644
--- a/llvm/test/CodeGen/AMDGPU/commute-compares-scalar-float.ll
+++ b/llvm/test/CodeGen/AMDGPU/commute-compares-scalar-float.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.50 -amdgpu-sdwa-peephole=0 < %s | FileCheck -check-prefix=SDAG %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.50 -amdgpu-sdwa-peephole=0 < %s | FileCheck -check-prefix=GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu11.50 -amdgpu-sdwa-peephole=0 < %s | FileCheck -check-prefix=GISEL %s
define amdgpu_vs void @fcmp_f32_olt_to_ogt(ptr addrspace(1) inreg %out, float inreg %a) {
; SDAG-LABEL: fcmp_f32_olt_to_ogt:
@@ -223,10 +223,11 @@ define amdgpu_vs void @fcmp_f16_olt_to_ogt(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_olt_to_ogt:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_gt_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp olt half 2.0, %a
@@ -248,10 +249,11 @@ define amdgpu_vs void @fcmp_f16_ogt_to_olt(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ogt_to_olt:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_lt_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ogt half 2.0, %a
@@ -273,10 +275,11 @@ define amdgpu_vs void @fcmp_f16_ole_to_oge(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ole_to_oge:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_ge_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ole half 2.0, %a
@@ -298,10 +301,11 @@ define amdgpu_vs void @fcmp_f16_oge_to_ole(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_oge_to_ole:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_le_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp oge half 2.0, %a
@@ -323,10 +327,11 @@ define amdgpu_vs void @fcmp_f16_ult_to_ugt(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ult_to_ugt:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_nle_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ult half 2.0, %a
@@ -348,10 +353,11 @@ define amdgpu_vs void @fcmp_f16_ugt_to_ult(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ugt_to_ult:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_nge_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ugt half 2.0, %a
@@ -373,10 +379,11 @@ define amdgpu_vs void @fcmp_ule_to_uge(ptr addrspace(1) inreg %out, half inreg %
; GISEL-LABEL: fcmp_ule_to_uge:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_nlt_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ule half 2.0, %a
@@ -398,10 +405,11 @@ define amdgpu_vs void @fcmp_uge_to_ule(ptr addrspace(1) inreg %out, half inreg %
; GISEL-LABEL: fcmp_uge_to_ule:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_ngt_f16 s2, 0x4000
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp uge half 2.0, %a
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index 37ab5cccea33b..d9b6edf00b82e 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -3,7 +3,7 @@
; RUN: llc < %s -mtriple=amdgpu8.02 -mattr=-flat-for-global | FileCheck %s -enable-var-scope --check-prefix=VI
; RUN: llc < %s -mtriple=r600 -mcpu=cypress | FileCheck %s -enable-var-scope --check-prefix=EG
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10
-; RUN: llc < %s -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
+; RUN: llc < %s -global-isel -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 | FileCheck %s -enable-var-scope --check-prefixes=GFX11,GFX11-TRUE16
; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 | FileCheck %s -enable-var-scope --check-prefixes=GFX11,GFX11-FAKE16
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop64.ll b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
index b0f2d87466707..edf9ef109a42d 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=SI %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index 1b964d35aadd9..81227d358ad29 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -3,7 +3,7 @@
; RUN: llc < %s -mtriple=amdgpu8.02 -mattr=-flat-for-global | FileCheck %s -enable-var-scope --check-prefix=VI
; RUN: llc < %s -mtriple=r600 -mcpu=cypress | FileCheck %s -enable-var-scope --check-prefix=EG
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10
-; RUN: llc < %s -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
+; RUN: llc < %s -global-isel -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
declare i7 @llvm.cttz.i7(i7, i1) nounwind readnone
declare i8 @llvm.cttz.i8(i8, i1) nounwind readnone
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index 1a00485907a47..cc72f3e342fdc 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -1996,24 +1996,43 @@ define half @fmul_select_f16_test6(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_f16_test6:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x4200
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xc800
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_f16_test6:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x4200
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xc800
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_f16_test6:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xc800
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x4200, v3, vcc_lo
-; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: fmul_select_f16_test6:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xc800
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0x4200
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_f16_test6:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0xc800
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x4200, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_f16_test6:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x4200
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc800, vcc_lo
+; GFX10-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test6:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -2048,10 +2067,10 @@ define half @fmul_select_f16_test6(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-GISEL-FAKE16-LABEL: fmul_select_f16_test6:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xc800
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x4200
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x4200, v3, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0xc800, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
@@ -2087,24 +2106,43 @@ define half @fmul_select_f16_test7(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_f16_test7:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xc400
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x4800
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_f16_test7:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0xc400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x4800
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_f16_test7:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x4800
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xc400, v3, vcc_lo
-; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: fmul_select_f16_test7:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x4800
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xc400
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_f16_test7:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x4800
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0xc400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_f16_test7:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xc400
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4800, vcc_lo
+; GFX10-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test7:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -2139,10 +2177,10 @@ define half @fmul_select_f16_test7(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-GISEL-FAKE16-LABEL: fmul_select_f16_test7:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x4800
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xc400
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xc400, v3, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0x4800, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
@@ -2324,26 +2362,42 @@ define half @fmul_select_f16_test10_sel_log2val_neg11_pos11(half %x, i32 %bool.a
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v1, v1, s4, v2
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: s_movk_i32 s4, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
-; GFX10-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
-; GFX10-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
+; GFX10-SDAG-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
+; GFX10-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -2380,10 +2434,8 @@ define half @fmul_select_f16_test10_sel_log2val_neg11_pos11(half %x, i32 %bool.a
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v1, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
@@ -2403,26 +2455,42 @@ define half @fmul_select_f16_test11_sel_log2val_pos7_neg14(half %x, i32 %bool.ar
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v1, v1, s4, v2
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: s_movk_i32 s4, 0x8000
-; GFX10-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
-; GFX10-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
-; GFX10-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
+; GFX10-SDAG-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
+; GFX10-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -2459,10 +2527,8 @@ define half @fmul_select_f16_test11_sel_log2val_pos7_neg14(half %x, i32 %bool.ar
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v1, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
diff --git a/llvm/test/CodeGen/AMDGPU/diverge-interp-mov-lower.ll b/llvm/test/CodeGen/AMDGPU/diverge-interp-mov-lower.ll
index ea715f82f073c..e0980f4658c25 100644
--- a/llvm/test/CodeGen/AMDGPU/diverge-interp-mov-lower.ll
+++ b/llvm/test/CodeGen/AMDGPU/diverge-interp-mov-lower.ll
@@ -1,9 +1,9 @@
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu8.02--amdpal | FileCheck --check-prefix=GCN %s
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02--amdpal | FileCheck --check-prefix=GCN %s
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu8.02--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu8.10--amdpal | FileCheck --check-prefix=GCN %s
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.10--amdpal | FileCheck --check-prefix=GCN %s
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu8.10--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu9.00--amdpal | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00--amdpal | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu9.00--amdpal | FileCheck -check-prefixes=GCN,GFX9 %s
; Testing for failures in divergence calculations when divergent intrinsic is lowered during instruction selection
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 52a5e242ea54b..028ec031941d3 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX11-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX11-GISEL %s
diff --git a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
index 6f921981ac212..9b6cec0c6d719 100644
--- a/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fadd.f16.ll
@@ -2,9 +2,9 @@
; RUN: llc -mtriple=amdgpu6.00 -enable-misched=false < %s | FileCheck -check-prefixes=SI %s
; RUN: llc -mtriple=amdgpu8.03 -mattr=-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=VI %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 -mattr=+real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-SDAG %s
-; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 -mattr=+real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-GISEL %s
+; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -mattr=+real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 -mattr=-real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-FAKE16-SDAG %s
-; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 -mattr=-real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-FAKE16-GISEL %s
+; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -mattr=-real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-FAKE16-GISEL %s
define amdgpu_kernel void @fadd_f16(
; SI-LABEL: fadd_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
index 7ddd5a5842bcc..830e53e2d2708 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16 %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-REAL16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-REAL16 %s
@@ -1372,6 +1372,11 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
+; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%data.hi = extractelement <2 x i16> %data, i32 1
@@ -1387,6 +1392,11 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
+; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1428,7 +1438,6 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg12
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1250-GISEL-FAKE16: {{.*}}
; GFX1250-GISEL-REAL16: {{.*}}
; GFX1250-SDAG-FAKE16: {{.*}}
; GFX1250-SDAG-REAL16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll b/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
index 68dd9144c9c26..40c256b106a2a 100644
--- a/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.50 -global-isel=0 < %s | FileCheck -check-prefix=SDAG-GFX1150 %s
-; RUN: llc -mtriple=amdgpu11.50 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GISEL-GFX1150 %s
+; RUN: llc -mtriple=amdgpu11.50 -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX1150 %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefix=SDAG-GFX1200 %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GISEL-GFX1200 %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX1200 %s
define amdgpu_vs void @f32_olt(ptr addrspace(1) inreg %out, float inreg %a, float inreg %b) {
; SDAG-GFX1150-LABEL: f32_olt:
@@ -647,10 +647,11 @@ define amdgpu_vs void @f16_olt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_olt:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_lt_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_olt:
@@ -665,10 +666,11 @@ define amdgpu_vs void @f16_olt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_olt:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_lt_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp olt half %a, %b
@@ -690,10 +692,11 @@ define amdgpu_vs void @f16_oeq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_oeq:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_eq_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_oeq:
@@ -708,10 +711,11 @@ define amdgpu_vs void @f16_oeq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_oeq:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_eq_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp oeq half %a, %b
@@ -733,10 +737,11 @@ define amdgpu_vs void @f16_ole(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ole:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_le_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ole:
@@ -751,10 +756,11 @@ define amdgpu_vs void @f16_ole(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ole:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_le_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ole half %a, %b
@@ -776,10 +782,11 @@ define amdgpu_vs void @f16_ogt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ogt:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_gt_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ogt:
@@ -794,10 +801,11 @@ define amdgpu_vs void @f16_ogt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ogt:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_gt_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ogt half %a, %b
@@ -819,10 +827,11 @@ define amdgpu_vs void @f16_one(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_one:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_lg_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_one:
@@ -837,10 +846,11 @@ define amdgpu_vs void @f16_one(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_one:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_lg_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp one half %a, %b
@@ -862,10 +872,11 @@ define amdgpu_vs void @f16_oge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_oge:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_ge_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_oge:
@@ -880,10 +891,11 @@ define amdgpu_vs void @f16_oge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_oge:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_ge_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp oge half %a, %b
@@ -905,10 +917,11 @@ define amdgpu_vs void @f16_ord(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ord:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_o_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ord:
@@ -923,10 +936,11 @@ define amdgpu_vs void @f16_ord(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ord:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_o_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ord half %a, %b
@@ -948,10 +962,11 @@ define amdgpu_vs void @f16_uno(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_uno:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_u_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_uno:
@@ -966,10 +981,11 @@ define amdgpu_vs void @f16_uno(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_uno:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_u_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp uno half %a, %b
@@ -991,10 +1007,11 @@ define amdgpu_vs void @f16_ult(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ult:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nge_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ult:
@@ -1009,10 +1026,11 @@ define amdgpu_vs void @f16_ult(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ult:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nge_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ult half %a, %b
@@ -1034,10 +1052,11 @@ define amdgpu_vs void @f16_ueq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ueq:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nlg_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ueq:
@@ -1052,10 +1071,11 @@ define amdgpu_vs void @f16_ueq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ueq:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nlg_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ueq half %a, %b
@@ -1077,10 +1097,11 @@ define amdgpu_vs void @f16_ule(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ule:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_ngt_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ule:
@@ -1095,10 +1116,11 @@ define amdgpu_vs void @f16_ule(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ule:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_ngt_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ule half %a, %b
@@ -1120,10 +1142,11 @@ define amdgpu_vs void @f16_ugt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ugt:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nle_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ugt:
@@ -1138,10 +1161,11 @@ define amdgpu_vs void @f16_ugt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ugt:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nle_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ugt half %a, %b
@@ -1163,10 +1187,11 @@ define amdgpu_vs void @f16_une(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_une:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_neq_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_une:
@@ -1181,10 +1206,11 @@ define amdgpu_vs void @f16_une(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_une:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_neq_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp une half %a, %b
@@ -1206,10 +1232,11 @@ define amdgpu_vs void @f16_uge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_uge:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nlt_f16 s2, s3
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_uge:
@@ -1224,10 +1251,11 @@ define amdgpu_vs void @f16_uge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_uge:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nlt_f16 s2, s3
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp uge half %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index 977a71012abe5..1aaed254c2fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -178,12 +178,19 @@ define half @test_fmac(half %x, half %y, half %z) {
; GFX10+ has v_fmaak_f16.
define half @test_fmaak(half %x, half %y, half %z) {
-; GFX9-LABEL: test_fmaak:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x4200
-; GFX9-NEXT: v_fma_f16 v0, v0, v1, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_fmaak:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x4200
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, v1, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_fmaak:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4200
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_fmaak:
; GFX10: ; %bb.0:
@@ -260,12 +267,19 @@ define half @test_fmaak(half %x, half %y, half %z) {
; GFX10+ has v_fmamk_f16.
define half @test_fmamk(half %x, half %y, half %z) {
-; GFX9-LABEL: test_fmamk:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x4200
-; GFX9-NEXT: v_fma_f16 v0, v0, s4, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_fmamk:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x4200
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_fmamk:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x4200
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_fmamk:
; GFX10: ; %bb.0:
@@ -342,28 +356,53 @@ define half @test_fmamk(half %x, half %y, half %z) {
; Regression test for a crash caused by D139469.
define i32 @test_D139469_f16(half %arg) {
-; GFX9-LABEL: test_D139469_f16:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x291e
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x211e
-; GFX9-NEXT: v_mul_f16_e32 v1, 0x291e, v0
-; GFX9-NEXT: v_fma_f16 v0, v0, s4, v2
-; GFX9-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX9-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_D139469_f16:
-; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_movk_i32 s4, 0x291e
-; GFX10-NEXT: v_mul_f16_e32 v1, 0x291e, v0
-; GFX10-NEXT: v_fmaak_f16 v0, s4, v0, 0x211e
-; GFX10-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX10-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_D139469_f16:
+; GFX9-SDAG: ; %bb.0: ; %bb
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x291e
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x211e
+; GFX9-SDAG-NEXT: v_mul_f16_e32 v1, 0x291e, v0
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, v2
+; GFX9-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_D139469_f16:
+; GFX9-GISEL: ; %bb.0: ; %bb
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mul_f16_e32 v2, 0x291e, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x291e
+; GFX9-GISEL-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x211e
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: v_cmp_gt_f16_e64 s[4:5], 0, v0
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_D139469_f16:
+; GFX10-SDAG: ; %bb.0: ; %bb
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x291e
+; GFX10-SDAG-NEXT: v_mul_f16_e32 v1, 0x291e, v0
+; GFX10-SDAG-NEXT: v_fmaak_f16 v0, s4, v0, 0x211e
+; GFX10-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX10-SDAG-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_D139469_f16:
+; GFX10-GISEL: ; %bb.0: ; %bb
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x211e
+; GFX10-GISEL-NEXT: v_mul_f16_e32 v2, 0x291e, v0
+; GFX10-GISEL-NEXT: v_fmac_f16_e32 v1, 0x291e, v0
+; GFX10-GISEL-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
+; GFX10-GISEL-NEXT: v_cmp_gt_f16_e64 s4, 0, v1
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_D139469_f16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
@@ -408,13 +447,15 @@ define i32 @test_D139469_f16(half %arg) {
; GFX11-GISEL-FAKE16-LABEL: test_D139469_f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x291e
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x291e, v0
-; GFX11-GISEL-FAKE16-NEXT: v_fmaak_f16 v0, s0, v0, 0x211e
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x211e
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x291e, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v1, 0x291e, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s0, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_D139469_f16:
@@ -480,15 +521,16 @@ define i32 @test_D139469_f16(half %arg) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x291e
-; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x291e, v0
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x211e
+; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x291e, v0
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v1, 0x291e, v0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s0, 0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-FAKE16-NEXT: v_fmaak_f16 v0, s0, v0, 0x211e
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v1, v0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
-; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
bb:
%i = fmul contract half %arg, 0xH291E
@@ -698,8 +740,3 @@ bb:
%i5 = zext <2 x i1> %i4 to <2 x i32>
ret <2 x i32> %i5
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10-GISEL: {{.*}}
-; GFX10-SDAG: {{.*}}
-; GFX9-GISEL: {{.*}}
-; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 64502fc2b3c11..6f07c945e68a2 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -3,12 +3,12 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define amdgpu_ps float @test_fmaximum_f32_vv(float %a, float %b) {
; GFX9-LABEL: test_fmaximum_f32_vv:
@@ -445,14 +445,25 @@ define amdgpu_ps half @test_fmaximum_f16_vv(half %a, half %b) {
}
define amdgpu_ps half @test_fmaximum_f16_ss(half inreg %a, half inreg %b) {
-; GFX9-LABEL: test_fmaximum_f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_max_f16_e32 v1, s0, v0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fmaximum_f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-SDAG-NEXT: v_max_f16_e32 v1, s0, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fmaximum_f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, s0, v0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s1, 0x7e00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-TRUE16-LABEL: test_fmaximum_f16_ss:
; GFX1170-SDAG-TRUE16: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index fc3ee5b731322..5e3e2ca025bc7 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -2,9 +2,9 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL %s
define amdgpu_kernel void @test_fmax_f32_ieee_mode_on(ptr addrspace(1) %out, float %a, float %b) #0 {
; GFX8-SDAG-LABEL: test_fmax_f32_ieee_mode_on:
@@ -1859,47 +1859,94 @@ define <3 x float> @test_func_fmax_v3f32(<3 x float> %a, <3 x float> %b) #0 {
}
define amdgpu_kernel void @test_fmax_f16_v_ieee_on(ptr addrspace(1) %out, half %a, half %b) #0 {
-; GFX8-LABEL: test_fmax_f16_v_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s6, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX8-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX8-NEXT: v_max_f16_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmax_f16_v_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: test_fmax_f16_v_ieee_on:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX9-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX9-NEXT: v_max_f16_e32 v0, v1, v0
-; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX9-NEXT: s_endpgm
-;
-; GFX12-LABEL: test_fmax_f16_v_ieee_on:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-NEXT: v_max_num_f16_e64 v0.h, s2, s2
-; GFX12-NEXT: v_max_num_f16_e64 v0.l, s3, s3
-; GFX12-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-NEXT: s_mov_b32 s2, -1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f16_e32 v0.l, v0.h, v0.l
-; GFX12-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-NEXT: s_endpgm
+; GFX8-GISEL-LABEL: test_fmax_f16_v_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
+; GFX8-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: s_lshr_b32 s4, s3, 16
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s3, s3
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: test_fmax_f16_v_ieee_on:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: test_fmax_f16_v_ieee_on:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-GISEL-NEXT: v_max_f16_e64 v0, s3, s3
+; GFX9-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX9-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: test_fmax_f16_v_ieee_on:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s3, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f16_e32 v0.l, v0.h, v0.l
+; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: test_fmax_f16_v_ieee_on:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v1.l, s3, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX12-GISEL-NEXT: s_max_num_f16 s2, s2, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-GISEL-NEXT: s_endpgm
%val = call half @llvm.maxnum.f16(half %a, half %b)
store half %val, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll b/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll
index c5f2f92636488..3031c4835ce10 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll
@@ -832,25 +832,45 @@ define half @fmed3_f32_fpext_f16_unrepresentable_k0(half %arg1, half %arg2) #1 {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: fmed3_f32_fpext_f16_unrepresentable_k0:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX8-NEXT: s_mov_b32 s4, 0x4f800000
-; GFX8-NEXT: v_med3_f32 v0, s4, v0, v1
-; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: fmed3_f32_fpext_f16_unrepresentable_k0:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9-NEXT: s_mov_b32 s4, 0x4f800000
-; GFX9-NEXT: v_med3_f32 v0, s4, v0, v1
-; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: fmed3_f32_fpext_f16_unrepresentable_k0:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX8-SDAG-NEXT: s_mov_b32 s4, 0x4f800000
+; GFX8-SDAG-NEXT: v_med3_f32 v0, s4, v0, v1
+; GFX8-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: fmed3_f32_fpext_f16_unrepresentable_k0:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 0x4f800000
+; GFX8-GISEL-NEXT: v_med3_f32 v0, v2, v0, v1
+; GFX8-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: fmed3_f32_fpext_f16_unrepresentable_k0:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x4f800000
+; GFX9-SDAG-NEXT: v_med3_f32 v0, s4, v0, v1
+; GFX9-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmed3_f32_fpext_f16_unrepresentable_k0:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4f800000
+; GFX9-GISEL-NEXT: v_med3_f32 v0, v2, v0, v1
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%arg1.ext = fpext half %arg1 to float
%arg2.ext = fpext half %arg2 to float
%med3 = call float @llvm.amdgcn.fmed3.f32(float 0x41f0000000000000, float %arg1.ext, float %arg2.ext)
@@ -879,25 +899,45 @@ define half @fmed3_f32_fpext_f16_unrepresentable_k1(half %arg0, half %arg2) #1 {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: fmed3_f32_fpext_f16_unrepresentable_k1:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX8-NEXT: s_mov_b32 s4, 0x4f800000
-; GFX8-NEXT: v_med3_f32 v0, v0, s4, v1
-; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: fmed3_f32_fpext_f16_unrepresentable_k1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9-NEXT: s_mov_b32 s4, 0x4f800000
-; GFX9-NEXT: v_med3_f32 v0, v0, s4, v1
-; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: fmed3_f32_fpext_f16_unrepresentable_k1:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX8-SDAG-NEXT: s_mov_b32 s4, 0x4f800000
+; GFX8-SDAG-NEXT: v_med3_f32 v0, v0, s4, v1
+; GFX8-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: fmed3_f32_fpext_f16_unrepresentable_k1:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 0x4f800000
+; GFX8-GISEL-NEXT: v_med3_f32 v0, v0, v2, v1
+; GFX8-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: fmed3_f32_fpext_f16_unrepresentable_k1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x4f800000
+; GFX9-SDAG-NEXT: v_med3_f32 v0, v0, s4, v1
+; GFX9-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmed3_f32_fpext_f16_unrepresentable_k1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4f800000
+; GFX9-GISEL-NEXT: v_med3_f32 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%arg0.ext = fpext half %arg0 to float
%arg2.ext = fpext half %arg2 to float
%med3 = call float @llvm.amdgcn.fmed3.f32(float %arg0.ext, float 0x41f0000000000000, float %arg2.ext)
@@ -926,25 +966,45 @@ define half @fmed3_f32_fpext_f16_unrepresentable_k2(half %arg0, half %arg1) #1 {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: fmed3_f32_fpext_f16_unrepresentable_k2:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX8-NEXT: s_mov_b32 s4, 0x4f800000
-; GFX8-NEXT: v_med3_f32 v0, v0, v1, s4
-; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: fmed3_f32_fpext_f16_unrepresentable_k2:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9-NEXT: s_mov_b32 s4, 0x4f800000
-; GFX9-NEXT: v_med3_f32 v0, v0, v1, s4
-; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: fmed3_f32_fpext_f16_unrepresentable_k2:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX8-SDAG-NEXT: s_mov_b32 s4, 0x4f800000
+; GFX8-SDAG-NEXT: v_med3_f32 v0, v0, v1, s4
+; GFX8-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: fmed3_f32_fpext_f16_unrepresentable_k2:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 0x4f800000
+; GFX8-GISEL-NEXT: v_med3_f32 v0, v0, v1, v2
+; GFX8-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: fmed3_f32_fpext_f16_unrepresentable_k2:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x4f800000
+; GFX9-SDAG-NEXT: v_med3_f32 v0, v0, v1, s4
+; GFX9-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmed3_f32_fpext_f16_unrepresentable_k2:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4f800000
+; GFX9-GISEL-NEXT: v_med3_f32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%arg0.ext = fpext half %arg0 to float
%arg1.ext = fpext half %arg1 to float
%med3 = call float @llvm.amdgcn.fmed3.f32(float %arg0.ext, float %arg1.ext, float 0x41f0000000000000)
@@ -954,8 +1014,3 @@ define half @fmed3_f32_fpext_f16_unrepresentable_k2(half %arg0, half %arg1) #1 {
attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX8-GISEL: {{.*}}
-; GFX8-SDAG: {{.*}}
-; GFX9-GISEL: {{.*}}
-; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.ll b/llvm/test/CodeGen/AMDGPU/fmed3.ll
index 9cdff09cd78d7..9b0d10061d170 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.ll
@@ -4,9 +4,9 @@
; RUN: llc -mtriple=amdgpu8.02 -global-isel=0 < %s | FileCheck -enable-var-scope -check-prefixes=VI-SDAG %s
; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope -check-prefixes=VI-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
@@ -9071,8 +9071,12 @@ define half @v_test_nnan_input_fmed3_r_i_i_f16_maximum_minimum(half %a) {
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-GISEL-NEXT: v_add_f16_e32 v0, 1.0, v0
-; VI-GISEL-NEXT: v_max_f16_e32 v0, 2.0, v0
-; VI-GISEL-NEXT: v_min_f16_e32 v0, 4.0, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_max_f32_e32 v0, 2.0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_min_f32_e32 v0, 4.0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_test_nnan_input_fmed3_r_i_i_f16_maximum_minimum:
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index a4baa88cdd08d..1a7707f0d261a 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -3,12 +3,12 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define amdgpu_ps float @test_fminimum_f32_vv(float %a, float %b) {
; GFX9-LABEL: test_fminimum_f32_vv:
@@ -445,14 +445,25 @@ define amdgpu_ps half @test_fminimum_f16_vv(half %a, half %b) {
}
define amdgpu_ps half @test_fminimum_f16_ss(half inreg %a, half inreg %b) {
-; GFX9-LABEL: test_fminimum_f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_min_f16_e32 v1, s0, v0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fminimum_f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-SDAG-NEXT: v_min_f16_e32 v1, s0, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fminimum_f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v1, s0, v0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s1, 0x7e00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-TRUE16-LABEL: test_fminimum_f16_ss:
; GFX1170-SDAG-TRUE16: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index 90a5d935d014c..7a58abb97d531 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -2,9 +2,9 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-GISEL %s
define amdgpu_kernel void @test_fmin_f32_ieee_mode_on(ptr addrspace(1) %out, float %a, float %b) #0 {
; GFX8-SDAG-LABEL: test_fmin_f32_ieee_mode_on:
@@ -1772,47 +1772,94 @@ define <3 x float> @test_func_fmin_v3f32(<3 x float> %a, <3 x float> %b) nounwin
}
define amdgpu_kernel void @test_fmin_f16_v_ieee_on(ptr addrspace(1) %out, half %a, half %b) #0 {
-; GFX8-LABEL: test_fmin_f16_v_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s6, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX8-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX8-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmin_f16_v_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: test_fmin_f16_v_ieee_on:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX9-NEXT: v_max_f16_e64 v1, s6, s6
-; GFX9-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX9-NEXT: s_endpgm
-;
-; GFX12-LABEL: test_fmin_f16_v_ieee_on:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-NEXT: v_max_num_f16_e64 v0.h, s2, s2
-; GFX12-NEXT: v_max_num_f16_e64 v0.l, s3, s3
-; GFX12-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-NEXT: s_mov_b32 s2, -1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f16_e32 v0.l, v0.h, v0.l
-; GFX12-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-NEXT: s_endpgm
+; GFX8-GISEL-LABEL: test_fmin_f16_v_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
+; GFX8-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: s_lshr_b32 s4, s3, 16
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s3, s3
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: test_fmin_f16_v_ieee_on:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s4, s6, 16
+; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
+; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX9-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: test_fmin_f16_v_ieee_on:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-GISEL-NEXT: v_max_f16_e64 v0, s3, s3
+; GFX9-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX9-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: test_fmin_f16_v_ieee_on:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s3, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f16_e32 v0.l, v0.h, v0.l
+; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: test_fmin_f16_v_ieee_on:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-GISEL-NEXT: v_max_num_f16_e64 v1.l, s3, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX12-GISEL-NEXT: s_min_num_f16 s2, s2, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX12-GISEL-NEXT: s_endpgm
%val = call half @llvm.minnum.f16(half %a, half %b)
store half %val, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 19c582dfcc0d1..153b1ffbeca46 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -3377,17 +3377,29 @@ define half @v_mul_1_f16(half %x) {
}
define half @v_mul_2_f16(half %x) {
-; GFX9-LABEL: v_mul_2_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_f16_e32 v0, v0, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_2_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_add_f16_e32 v0, v0, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_mul_2_f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_f16_e32 v0, v0, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_2_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mul_f16_e32 v0, 2.0, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_2_f16:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_add_f16_e32 v0, v0, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_2_f16:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mul_f16_e32 v0, 2.0, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_2_f16:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -3410,7 +3422,7 @@ define half @v_mul_2_f16(half %x) {
; GFX11-GISEL-FAKE16-LABEL: v_mul_2_f16:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v0
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, 2.0, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%mul = fmul half %x, 2.0
ret half %mul
@@ -3780,12 +3792,19 @@ define half @v_mul_0x1p14_f16(half %x) {
; Check that this doesn't interfer with fma formation
define half @v_fma_mul_add_32_f16(half %x, half %y) {
-; GFX9-LABEL: v_fma_mul_add_32_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x5000
-; GFX9-NEXT: v_fma_f16 v0, v0, s4, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_fma_mul_add_32_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x5000
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: v_fma_mul_add_32_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x5000
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fma_mul_add_32_f16:
; GFX10: ; %bb.0:
@@ -3822,12 +3841,19 @@ define half @v_fma_mul_add_32_f16(half %x, half %y) {
}
define half @v_fma_mul_sub_32_f16(half %x, half %y) {
-; GFX9-LABEL: v_fma_mul_sub_32_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x5000
-; GFX9-NEXT: v_fma_f16 v0, v0, s4, -v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_fma_mul_sub_32_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x5000
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, -v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: v_fma_mul_sub_32_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x5000
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v2, -v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fma_mul_sub_32_f16:
; GFX10: ; %bb.0:
@@ -3864,12 +3890,19 @@ define half @v_fma_mul_sub_32_f16(half %x, half %y) {
}
define half @v_fma_mul_add_neg32_f16(half %x, half %y) {
-; GFX9-LABEL: v_fma_mul_add_neg32_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, 0xd000
-; GFX9-NEXT: v_fma_f16 v0, v0, s4, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_fma_mul_add_neg32_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0xd000
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: v_fma_mul_add_neg32_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xd000
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fma_mul_add_neg32_f16:
; GFX10: ; %bb.0:
@@ -3906,12 +3939,19 @@ define half @v_fma_mul_add_neg32_f16(half %x, half %y) {
}
define half @v_mul_fabs_32_f16(half %x) {
-; GFX9-LABEL: v_mul_fabs_32_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x5000
-; GFX9-NEXT: v_mul_f16_e64 v0, |v0|, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_fabs_32_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x5000
+; GFX9-SDAG-NEXT: v_mul_f16_e64 v0, |v0|, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: v_mul_fabs_32_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x5000
+; GFX9-GISEL-NEXT: v_mul_f16_e64 v0, |v0|, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_fabs_32_f16:
; GFX10: ; %bb.0:
@@ -3948,12 +3988,19 @@ define half @v_mul_fabs_32_f16(half %x) {
}
define half @v_mul_add_fma_fabs_32_f16(half %x, half %y) {
-; GFX9-LABEL: v_mul_add_fma_fabs_32_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x5000
-; GFX9-NEXT: v_fma_f16 v0, |v0|, s4, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_add_fma_fabs_32_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x5000
+; GFX9-SDAG-NEXT: v_fma_f16 v0, |v0|, s4, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: v_mul_add_fma_fabs_32_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x5000
+; GFX9-GISEL-NEXT: v_fma_f16 v0, |v0|, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_fma_fabs_32_f16:
; GFX10: ; %bb.0:
@@ -4167,19 +4214,34 @@ define <2 x half> @v_fma_mul_add_32_v2f16(<2 x half> %x, <2 x half> %y) {
}
define amdgpu_ps i32 @s_mul_32_f16(half inreg %x, half inreg %y) {
-; GFX9-LABEL: s_mul_32_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5000
-; GFX9-NEXT: v_mul_f16_e32 v0, s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: s_mul_32_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5000
+; GFX9-SDAG-NEXT: v_mul_f16_e32 v0, s0, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: s_mul_32_f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mul_f16_e64 v0, 0x5000, s0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
+; GFX9-GISEL-LABEL: s_mul_32_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5000
+; GFX9-GISEL-NEXT: v_mul_f16_e32 v0, s0, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX10-SDAG-LABEL: s_mul_32_f16:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_mul_f16_e64 v0, 0x5000, s0
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX10-GISEL-LABEL: s_mul_32_f16:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_mul_f16_e64 v0, 0x5000, s0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-GISEL-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-TRUE16-LABEL: s_mul_32_f16:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -4198,15 +4260,15 @@ define amdgpu_ps i32 @s_mul_32_f16(half inreg %x, half inreg %y) {
; GFX11-GISEL-TRUE16-LABEL: s_mul_32_f16:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e64 v0.l, 0x5000, s0
-; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-FAKE16-LABEL: s_mul_32_f16:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e64 v0, 0x5000, s0
-; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
; GFX11-GISEL-FAKE16-NEXT: ; return to shader part epilog
%mul = fmul contract half %x, 32.0
%cast = bitcast half %mul to i16
@@ -5928,24 +5990,42 @@ define half @v_mul_f16_select_n128_n16(i32 %arg, half %x) {
}
define half @v_contract_mul_add_f16_select_64_1(i32 %arg, half %x, half %y) {
-; GFX9-LABEL: v_contract_mul_add_f16_select_64_1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x3c00
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x5400
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_f16_select_64_1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x3c00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x5400
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_f16_select_64_1:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x5400
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v3, vcc_lo
-; GFX10-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_f16_select_64_1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_f16_select_64_1:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x5400
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_f16_select_64_1:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX10-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_64_1:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -5977,10 +6057,10 @@ define half @v_contract_mul_add_f16_select_64_1(i32 %arg, half %x, half %y) {
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_f16_select_64_1:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x5400
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq i32 %arg, 0
%select.pow2 = select contract i1 %cond, half 64.0, half 1.0
@@ -5990,24 +6070,42 @@ define half @v_contract_mul_add_f16_select_64_1(i32 %arg, half %x, half %y) {
}
define half @v_contract_mul_add_f16_select_1_64(i32 %arg, half %x, half %y) {
-; GFX9-LABEL: v_contract_mul_add_f16_select_1_64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x5400
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x3c00
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_f16_select_1_64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x5400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x3c00
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_f16_select_1_64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x3c00
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x5400, v3, vcc_lo
-; GFX10-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_f16_select_1_64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_f16_select_1_64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x3c00
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x5400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_f16_select_1_64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX10-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_1_64:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6039,10 +6137,10 @@ define half @v_contract_mul_add_f16_select_1_64(i32 %arg, half %x, half %y) {
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_f16_select_1_64:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x3c00
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x5400, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq i32 %arg, 0
%select.pow2 = select contract i1 %cond, half 1.0, half 64.0
@@ -6052,24 +6150,42 @@ define half @v_contract_mul_add_f16_select_1_64(i32 %arg, half %x, half %y) {
}
define half @v_contract_mul_add_f16_select_n64_n1(i32 %arg, half %x, half %y) {
-; GFX9-LABEL: v_contract_mul_add_f16_select_n64_n1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xbc00
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xd400
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_f16_select_n64_n1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0xbc00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xd400
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_f16_select_n64_n1:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xd400
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0xbc00, v3, vcc_lo
-; GFX10-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_f16_select_n64_n1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e64 v0, -v1, v0
+; GFX9-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_f16_select_n64_n1:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0xd400
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0xbc00, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_f16_select_n64_n1:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e64 v0, -v1, v0
+; GFX10-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_n64_n1:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6101,10 +6217,10 @@ define half @v_contract_mul_add_f16_select_n64_n1(i32 %arg, half %x, half %y) {
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_f16_select_n64_n1:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xd400
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0xbc00, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e64 v0, -v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq i32 %arg, 0
%select.pow2 = select contract i1 %cond, half -64.0, half -1.0
@@ -6114,24 +6230,42 @@ define half @v_contract_mul_add_f16_select_n64_n1(i32 %arg, half %x, half %y) {
}
define half @v_contract_mul_add_f16_select_n1_n64(i32 %arg, half %x, half %y) {
-; GFX9-LABEL: v_contract_mul_add_f16_select_n1_n64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xd400
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xbc00
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_f16_select_n1_n64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0xd400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xbc00
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_f16_select_n1_n64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xbc00
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0xd400, v3, vcc_lo
-; GFX10-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_f16_select_n1_n64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e64 v0, -v1, v0
+; GFX9-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_f16_select_n1_n64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0xbc00
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0xd400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_f16_select_n1_n64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e64 v0, -v1, v0
+; GFX10-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_n1_n64:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6163,10 +6297,10 @@ define half @v_contract_mul_add_f16_select_n1_n64(i32 %arg, half %x, half %y) {
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_f16_select_n1_n64:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xbc00
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0xd400, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e64 v0, -v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq i32 %arg, 0
%select.pow2 = select contract i1 %cond, half -1.0, half -64.0
@@ -6238,24 +6372,42 @@ define half @v_contract_mul_add_f16_select_128_64(i32 %arg, half %x, half %y) {
}
define half @v_contract_mul_add_f16_select_128_4(i32 %arg, half %x, half %y) {
-; GFX9-LABEL: v_contract_mul_add_f16_select_128_4:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x4400
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x5800
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_f16_select_128_4:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x4400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x5800
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_f16_select_128_4:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x5800
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x4400, v3, vcc_lo
-; GFX10-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_f16_select_128_4:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 2, 7, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_f16_select_128_4:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x5800
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x4400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_f16_select_128_4:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 2, 7, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX10-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_128_4:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6287,10 +6439,10 @@ define half @v_contract_mul_add_f16_select_128_4(i32 %arg, half %x, half %y) {
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_f16_select_128_4:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x5800
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x4400, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 2, 7, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq i32 %arg, 0
%select.pow2 = select i1 %cond, half 128.0, half 4.0
@@ -6362,24 +6514,42 @@ define half @v_contract_mul_add_f16_select_2_4(i32 %arg, half %x, half %y) {
}
define half @v_contract_mul_add_f16_select_4_128(i32 %arg, half %x, half %y) {
-; GFX9-LABEL: v_contract_mul_add_f16_select_4_128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x5800
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x4400
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_f16_select_4_128:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x5800
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x4400
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_f16_select_4_128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x4400
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x5800, v3, vcc_lo
-; GFX10-NEXT: v_fma_f16 v0, v1, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_f16_select_4_128:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 7, 2, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_f16_select_4_128:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x4400
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x5800, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_f16_select_4_128:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 7, 2, vcc_lo
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX10-GISEL-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_4_128:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6411,10 +6581,10 @@ define half @v_contract_mul_add_f16_select_4_128(i32 %arg, half %x, half %y) {
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_f16_select_4_128:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x4400
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x5800, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v1, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 7, 2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v1, v0
+; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq i32 %arg, 0
%select.pow2 = select i1 %cond, half 4.0, half 128.0
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index cc3da64f515de..56415ef44c527 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -8,13 +8,13 @@
; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck --check-prefixes=GFX11,GFX11-SDAG %s
; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck --check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.0a -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-GISEL,GFX90A-MUBUF,GFX90A-GISEL-MUBUF %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.0a -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-GISEL,GFX90A-FLATSCR,GFX90A-GISEL-FLATSCR %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-GISEL,GFX10-MUBUF,GFX10-GISEL-MUBUF %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-GISEL,GFX10-FLATSCR,GFX10-GISEL-FLATSCR %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 < %s | FileCheck --check-prefixes=GFX11,GFX11-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.0a -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-GISEL,GFX90A-MUBUF,GFX90A-GISEL-MUBUF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.0a -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-GISEL,GFX90A-FLATSCR,GFX90A-GISEL-FLATSCR %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30 -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-GISEL,GFX10-MUBUF,GFX10-GISEL-MUBUF %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-GISEL,GFX10-FLATSCR,GFX10-GISEL-FLATSCR %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 < %s | FileCheck --check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s
; This test checks memory addresses with constant offset components that should
; not be folded into memory accesses with immediate offsets.
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index d0482b6f7457b..7b4dcaa1c3dab 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -527,62 +527,121 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
}
define i128 @fptoui_bf16_to_i128(bfloat %x) {
-; GCN-LABEL: fptoui_bf16_to_i128:
-; GCN: ; %bb.0: ; %fp-to-i-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v4, v0
-; GCN-NEXT: v_bfe_u32 v5, v4, 7, 8
-; GCN-NEXT: s_movk_i32 s4, 0x7e
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: v_cmp_lt_u16_e32 vcc, s4, v5
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB7_6
-; GCN-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
-; GCN-NEXT: v_and_b32_e32 v0, 0x7f, v4
-; GCN-NEXT: s_movk_i32 s4, 0x85
-; GCN-NEXT: v_or_b32_e32 v4, 0x80, v0
-; GCN-NEXT: v_cmp_lt_u16_e32 vcc, s4, v5
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB7_3
-; GCN-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
-; GCN-NEXT: v_add_u16_e32 v6, 0xff7a, v5
-; GCN-NEXT: s_mov_b32 s4, 0
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff, v4
-; GCN-NEXT: v_mov_b32_e32 v1, s4
-; GCN-NEXT: v_sub_u32_e32 v2, 64, v6
-; GCN-NEXT: v_subrev_u32_e32 v4, 64, v6
-; GCN-NEXT: v_lshrrev_b64 v[2:3], v2, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[4:5], v4, v[0:1]
-; GCN-NEXT: v_cmp_gt_u16_e32 vcc, 64, v6
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GCN-NEXT: v_cmp_ne_u16_e64 s[4:5], 0, v6
-; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr5
-; GCN-NEXT: ; implicit-def: $vgpr4
-; GCN-NEXT: .LBB7_3: ; %Flow
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GCN-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
-; GCN-NEXT: v_sub_u16_e32 v0, 0x86, v5
-; GCN-NEXT: v_lshrrev_b16_e32 v0, v0, v4
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: ; %bb.5: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB7_6: ; %fp-to-i-cleanup
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fptoui_bf16_to_i128:
+; SDAG: ; %bb.0: ; %fp-to-i-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v4, v0
+; SDAG-NEXT: v_bfe_u32 v5, v4, 7, 8
+; SDAG-NEXT: s_movk_i32 s4, 0x7e
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s4, v5
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB7_6
+; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v4
+; SDAG-NEXT: s_movk_i32 s4, 0x85
+; SDAG-NEXT: v_or_b32_e32 v4, 0x80, v0
+; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s4, v5
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB7_3
+; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; SDAG-NEXT: v_add_u16_e32 v6, 0xff7a, v5
+; SDAG-NEXT: s_mov_b32 s4, 0
+; SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SDAG-NEXT: v_mov_b32_e32 v1, s4
+; SDAG-NEXT: v_sub_u32_e32 v2, 64, v6
+; SDAG-NEXT: v_subrev_u32_e32 v4, 64, v6
+; SDAG-NEXT: v_lshrrev_b64 v[2:3], v2, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[4:5], v4, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u16_e32 vcc, 64, v6
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; SDAG-NEXT: v_cmp_ne_u16_e64 s[4:5], 0, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr5
+; SDAG-NEXT: ; implicit-def: $vgpr4
+; SDAG-NEXT: .LBB7_3: ; %Flow
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; SDAG-NEXT: v_sub_u16_e32 v0, 0x86, v5
+; SDAG-NEXT: v_lshrrev_b16_e32 v0, v0, v4
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: ; %bb.5: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB7_6: ; %fp-to-i-cleanup
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fptoui_bf16_to_i128:
+; GISEL: ; %bb.0: ; %fp-to-i-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GISEL-NEXT: v_lshrrev_b16_e32 v5, 7, v4
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x7f
+; GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GISEL-NEXT: v_cmp_ge_u16_sdwa s[8:9], v5, v0 src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-NEXT: v_mov_b32_e32 v3, s7
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB7_6
+; GISEL-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v4
+; GISEL-NEXT: v_or_b32_e32 v4, 0x80, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x86
+; GISEL-NEXT: v_cmp_ge_u16_sdwa s[6:7], v5, v0 src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
+; GISEL-NEXT: s_xor_b64 s[6:7], exec, s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB7_3
+; GISEL-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; GISEL-NEXT: v_mov_b32_e32 v0, 0xffffff7a
+; GISEL-NEXT: v_add_u16_sdwa v8, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: v_add_u32_e32 v6, 0xffffffc0, v8
+; GISEL-NEXT: v_sub_u32_e32 v4, 64, v8
+; GISEL-NEXT: v_lshlrev_b64 v[2:3], v8, v[0:1]
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], v4, v[0:1]
+; GISEL-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8
+; GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr5
+; GISEL-NEXT: ; implicit-def: $vgpr4
+; GISEL-NEXT: .LBB7_3: ; %Flow
+; GISEL-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GISEL-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x86
+; GISEL-NEXT: v_sub_u16_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_lshrrev_b16_e32 v0, v0, v4
+; GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: ; %bb.5: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: .LBB7_6: ; %fp-to-i-cleanup
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%cvt = fptoui bfloat %x to i128
ret i128 %cvt
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
index 011cd696c3903..80f5989277b40 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-ISEL
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu9.00 | FileCheck %s --check-prefixes=GFX9
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-TRUE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-TRUE16
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
;
; 32-bit float to signed integer
@@ -2100,8 +2100,16 @@ define i1 @test_s_signed_i1_f16(half inreg %f) nounwind {
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-GI-NEXT: v_med3_i32 v0, v0, -1, 0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s0, s0, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i1 @llvm.fptosi.sat.i1.f16(half %f)
ret i1 %x
@@ -2193,10 +2201,16 @@ define i8 @test_s_signed_i8_f16(half inreg %f) nounwind {
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: s_movk_i32 s0, 0xff80
-; GFX12-GI-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7f
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_sext_i32_i16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_med3_i32 v0, v0, s0, 0x7f
+; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i8 @llvm.fptosi.sat.i8.f16(half %f)
ret i8 %x
@@ -2346,21 +2360,36 @@ define i64 @test_s_signed_i64_f16(half inreg %f) nounwind {
; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_s_signed_i64_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_ashr_i32 s1, s0, 31
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_s_signed_i64_f16:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-ISEL-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-ISEL-NEXT: s_ashr_i32 s1, s0, 31
+; GFX12-ISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_s_signed_i64_f16:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i64 @llvm.fptosi.sat.i64.f16(half %f)
ret i64 %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index 5ca3e294e417e..2622d77e27402 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -6,7 +6,7 @@
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-TRUE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-TRUE16
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
;
; Float to signed 32-bit -- Vector size variation
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
index 3aa3d282cbf9a..8b6df1ca1e69c 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-ISEL
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu9.00 | FileCheck %s --check-prefixes=GFX9
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-TRUE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-TRUE16
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
;
; 32-bit float to unsigned integer
@@ -1655,8 +1655,12 @@ define i1 @test_s_unsigned_i1_f16(half inreg %f) nounwind {
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i1 @llvm.fptoui.sat.i1.f16(half %f)
ret i1 %x
@@ -1737,8 +1741,12 @@ define i8 @test_s_unsigned_i8_f16(half inreg %f) nounwind {
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-GI-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX12-GI-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GI-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call i8 @llvm.fptoui.sat.i8.f16(half %f)
ret i8 %x
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 0ccf9de11e06a..4689a8e695f87 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -6,7 +6,7 @@
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-TRUE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-ISEL,GFX12-TRUE16
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-GI
;
; Float to unsigned 32-bit -- Vector size variation
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 3fcf85e5efe20..871c6e4e4458f 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -5220,19 +5220,16 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; SI-GISEL-LABEL: fptrunc_f32_to_f16_zext_i32:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; SI-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fptrunc_f32_to_f16_zext_i32:
@@ -5256,19 +5253,17 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; VI-GISEL-LABEL: fptrunc_f32_to_f16_zext_i32:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; VI-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_nop 1
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fptrunc_f32_to_f16_zext_i32:
@@ -5292,19 +5287,17 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; GFX9-GISEL-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fptrunc_f32_to_f16_zext_i32:
@@ -5328,19 +5321,18 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; GFX950-GISEL-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX950-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX950-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
@@ -5386,41 +5378,33 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; GFX11-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
@@ -5477,22 +5461,18 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_zext_i32:
@@ -5501,22 +5481,18 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) #0 {
@@ -5551,19 +5527,16 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; SI-GISEL-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; SI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; SI-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
@@ -5587,19 +5560,17 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; VI-GISEL-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; VI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; VI-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_nop 1
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
@@ -5623,19 +5594,17 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; GFX9-GISEL-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
@@ -5659,19 +5628,18 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; GFX950-GISEL-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; GFX950-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX950-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
@@ -5717,41 +5685,33 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; GFX11-GISEL-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, |v0|
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e64 v0.l, |s2|
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e64 v0, |s2|
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
@@ -5810,23 +5770,19 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_bitset0_b32 s2, 31
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1250-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
@@ -5835,23 +5791,19 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_bitset0_b32 s2, 31
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) #0 {
@@ -5888,20 +5840,16 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; SI-GISEL-LABEL: fptrunc_f32_to_f16_sext_i32:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16
-; SI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; SI-GISEL-NEXT: s_sext_i32_i16 s3, s3
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fptrunc_f32_to_f16_sext_i32:
@@ -5926,20 +5874,17 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; VI-GISEL-LABEL: fptrunc_f32_to_f16_sext_i32:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16
-; VI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; VI-GISEL-NEXT: s_sext_i32_i16 s3, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_nop 1
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fptrunc_f32_to_f16_sext_i32:
@@ -5964,20 +5909,17 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; GFX9-GISEL-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX9-GISEL-NEXT: s_mov_b32 s11, s7
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX9-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX9-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX9-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-GISEL-NEXT: s_sext_i32_i16 s3, s3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: fptrunc_f32_to_f16_sext_i32:
@@ -6002,20 +5944,18 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; GFX950-GISEL-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX950-GISEL: ; %bb.0: ; %entry
; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; GFX950-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX950-GISEL-NEXT: s_mov_b32 s10, s6
-; GFX950-GISEL-NEXT: s_mov_b32 s11, s7
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: s_mov_b32 s8, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s9, s3
-; GFX950-GISEL-NEXT: buffer_load_dword v0, off, s[8:11], 0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s0
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s1
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX950-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX950-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX950-GISEL-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX950-GISEL-NEXT: s_sext_i32_i16 s3, s3
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_sext_i32:
@@ -6061,41 +6001,33 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; GFX11-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX11-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_sext_i32:
@@ -6152,22 +6084,18 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-GISEL-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_sext_i32:
@@ -6176,22 +6104,18 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-GISEL-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll b/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
index 5877189952e05..fd5ea6eab210e 100644
--- a/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=CHECK,SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
; Test that fneg is folded into source modifiers when it wasn't
; possible to fold fsub to fneg without context.
@@ -339,66 +339,108 @@ define <2 x float> @fold_v2f32_fsub_into_fneg_modifier_dynamic_nsz(<2 x float> %
define half @fold_f16_fsub_into_fneg_modifier_ieee(half %v0, half %v1) #0 {
-; CHECK-LABEL: fold_f16_fsub_into_fneg_modifier_ieee:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mul_f16_e64 v0, -v0, v1
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fold_f16_fsub_into_fneg_modifier_ieee:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mul_f16_e64 v0, -v0, v1
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_ieee:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%mul = fmul half %sub, %v1
ret half %mul
}
define half @fold_f16_fsub_into_fneg_modifier_daz(half %v0, half %v1) #1 {
-; CHECK-LABEL: fold_f16_fsub_into_fneg_modifier_daz:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mul_f16_e64 v0, -v0, v1
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fold_f16_fsub_into_fneg_modifier_daz:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mul_f16_e64 v0, -v0, v1
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_daz:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%mul = fmul half %sub, %v1
ret half %mul
}
define half @fold_f16_fsub_into_fneg_modifier_ieee_nsz(half %v0, half %v1) #0 {
-; CHECK-LABEL: fold_f16_fsub_into_fneg_modifier_ieee_nsz:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mul_f16_e64 v0, -v0, v1
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fold_f16_fsub_into_fneg_modifier_ieee_nsz:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mul_f16_e64 v0, -v0, v1
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_ieee_nsz:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub nsz half -0.0, %v0
%mul = fmul nsz half %sub, %v1
ret half %mul
}
define half @fold_f16_fsub_into_fneg_modifier_daz_nsz(half %v0, half %v1) #1 {
-; CHECK-LABEL: fold_f16_fsub_into_fneg_modifier_daz_nsz:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mul_f16_e64 v0, -v0, v1
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fold_f16_fsub_into_fneg_modifier_daz_nsz:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mul_f16_e64 v0, -v0, v1
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_daz_nsz:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub nsz half -0.0, %v0
%mul = fmul nsz half %sub, %v1
ret half %mul
}
define half @fold_f16_fsub_into_fneg_modifier_dynamic(half %v0, half %v1) #2 {
-; CHECK-LABEL: fold_f16_fsub_into_fneg_modifier_dynamic:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mul_f16_e64 v0, -v0, v1
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fold_f16_fsub_into_fneg_modifier_dynamic:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mul_f16_e64 v0, -v0, v1
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_dynamic:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%mul = fmul half %sub, %v1
ret half %mul
}
define half @fold_f16_fsub_into_fneg_modifier_dynamic_nsz(half %v0, half %v1) #2 {
-; CHECK-LABEL: fold_f16_fsub_into_fneg_modifier_dynamic_nsz:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mul_f16_e64 v0, -v0, v1
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fold_f16_fsub_into_fneg_modifier_dynamic_nsz:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mul_f16_e64 v0, -v0, v1
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_dynamic_nsz:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub nsz half -0.0, %v0
%mul = fmul nsz half %sub, %v1
ret half %mul
@@ -689,42 +731,69 @@ define float @no_fold_f32_select_user_fsub_into_fneg_modifier_dynamic(i1 %cond,
}
define half @fold_f16_select_user_fsub_into_fneg_modifier_ieee(i1 %cond, half %v0, half %v1) #0 {
-; CHECK-LABEL: fold_f16_select_user_fsub_into_fneg_modifier_ieee:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fold_f16_select_user_fsub_into_fneg_modifier_ieee:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_and_b32_e32 v0, 1, v0
+; SDAG-NEXT: v_xor_b32_e32 v1, 0x8000, v1
+; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fold_f16_select_user_fsub_into_fneg_modifier_ieee:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-NEXT: v_max_f16_e64 v1, -v1, -v1
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%mul = select i1 %cond, half %sub, half %v1
ret half %mul
}
define half @no_fold_f16_select_user_fsub_into_fneg_modifier_daz(i1 %cond, half %v0, half %v1) #1 {
-; CHECK-LABEL: no_fold_f16_select_user_fsub_into_fneg_modifier_daz:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT: v_sub_f16_e32 v1, 0x8000, v1
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: no_fold_f16_select_user_fsub_into_fneg_modifier_daz:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_and_b32_e32 v0, 1, v0
+; SDAG-NEXT: v_sub_f16_e32 v1, 0x8000, v1
+; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: no_fold_f16_select_user_fsub_into_fneg_modifier_daz:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-NEXT: v_max_f16_e64 v1, -v1, -v1
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%mul = select i1 %cond, half %sub, half %v1
ret half %mul
}
define half @no_fold_f16_select_user_fsub_into_fneg_modifier_dynamic(i1 %cond, half %v0, half %v1) #2 {
-; CHECK-LABEL: no_fold_f16_select_user_fsub_into_fneg_modifier_dynamic:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT: v_sub_f16_e32 v1, 0x8000, v1
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: no_fold_f16_select_user_fsub_into_fneg_modifier_dynamic:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_and_b32_e32 v0, 1, v0
+; SDAG-NEXT: v_sub_f16_e32 v1, 0x8000, v1
+; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: no_fold_f16_select_user_fsub_into_fneg_modifier_dynamic:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-NEXT: v_max_f16_e64 v1, -v1, -v1
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%mul = select i1 %cond, half %sub, half %v1
ret half %mul
@@ -1123,13 +1192,21 @@ define i1 @no_fold_f64_fsub_into_fneg_modifier_class_var_daz(double %v0, i32 %te
}
define i1 @no_fold_f16_fsub_into_fneg_modifier_class_var_daz(half %v0, i32 %testmask) #1 {
-; CHECK-LABEL: no_fold_f16_fsub_into_fneg_modifier_class_var_daz:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_sub_f16_e32 v0, 0x8000, v0
-; CHECK-NEXT: v_cmp_class_f16_e32 vcc, v0, v1
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: no_fold_f16_fsub_into_fneg_modifier_class_var_daz:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_sub_f16_e32 v0, 0x8000, v0
+; SDAG-NEXT: v_cmp_class_f16_e32 vcc, v0, v1
+; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: no_fold_f16_fsub_into_fneg_modifier_class_var_daz:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_cmp_class_f16_e32 vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%class = call i1 @llvm.amdgcn.class.f16(half %sub, i32 %testmask)
ret i1 %class
@@ -1158,14 +1235,23 @@ define i1 @no_fold_f64_fsub_into_fneg_modifier_class_daz(double %v0) #1 {
}
define i1 @no_fold_f16_fsub_into_fneg_modifier_class_daz(half %v0) #1 {
-; CHECK-LABEL: no_fold_f16_fsub_into_fneg_modifier_class_daz:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_sub_f16_e32 v0, 0x8000, v0
-; CHECK-NEXT: v_mov_b32_e32 v1, 0x90
-; CHECK-NEXT: v_cmp_class_f16_e32 vcc, v0, v1
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: no_fold_f16_fsub_into_fneg_modifier_class_daz:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_sub_f16_e32 v0, 0x8000, v0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0x90
+; SDAG-NEXT: v_cmp_class_f16_e32 vcc, v0, v1
+; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: no_fold_f16_fsub_into_fneg_modifier_class_daz:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_max_f16_e64 v0, -v0, -v0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0x90
+; GISEL-NEXT: v_cmp_class_f16_e32 vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub half -0.0, %v0
%class = call i1 @llvm.is.fpclass.f16(half %sub, i32 144)
ret i1 %class
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index ef946c807eaa1..5aefa307f9117 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -15,17 +15,17 @@
; RUN: llc -global-isel=1 -mtriple=amdgpu9.0a-amd-amdpal < %s | FileCheck -check-prefixes=GFX9,GFX90A,GFX9-GISEL,GFX90A-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30-amd-amdpal < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30-amd-amdpal < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30-amd-amdpal < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1200,GFX1200-SDAG,GFX1200-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1200,GFX1200-SDAG,GFX1200-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1200,GFX1200-GISEL,GFX1200-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1200,GFX1200-GISEL,GFX1200-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1200,GFX1200-GISEL,GFX1200-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1200,GFX1200-GISEL,GFX1200-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-FAKE16,GFX1250-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdpal -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16,GFX1250-GISEL-FAKE16 %s
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
index 4a0ac7215806e..e99181f425be9 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
@@ -522,35 +522,37 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_float(float inreg %a, float %b)
define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
; GISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_half
- ; GISEL-GFX11: bb.0 (%ir-block.0):
+ ; GISEL-GFX11: bb.1 (%ir-block.0):
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
- ; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
- ; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
- ; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; GISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+ ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY3]]
+ ; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
+ ; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
; GISEL-GFX10-LABEL: name: amdgpu_cs_chain_cc_half
- ; GISEL-GFX10: bb.0 (%ir-block.0):
+ ; GISEL-GFX10: bb.1 (%ir-block.0):
; GISEL-GFX10-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX10-NEXT: {{ $}}
- ; GISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; GISEL-GFX10-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX10-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
- ; GISEL-GFX10-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
+ ; GISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; GISEL-GFX10-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY]]
+ ; GISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY1]]
; GISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
; GISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]]
- ; GISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]]
- ; GISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]]
- ; GISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1
- ; GISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX10-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
+ ; GISEL-GFX10-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
+ ; GISEL-GFX10-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_half
@@ -590,35 +592,37 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %b) {
; GISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; GISEL-GFX11: bb.0 (%ir-block.0):
+ ; GISEL-GFX11: bb.1 (%ir-block.0):
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
- ; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
- ; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
- ; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; GISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+ ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY3]]
+ ; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
+ ; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
; GISEL-GFX10-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; GISEL-GFX10: bb.0 (%ir-block.0):
+ ; GISEL-GFX10: bb.1 (%ir-block.0):
; GISEL-GFX10-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX10-NEXT: {{ $}}
- ; GISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; GISEL-GFX10-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX10-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
- ; GISEL-GFX10-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
+ ; GISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; GISEL-GFX10-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY]]
+ ; GISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY1]]
; GISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
; GISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]]
- ; GISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]]
- ; GISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]]
- ; GISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1
- ; GISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ ; GISEL-GFX10-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
+ ; GISEL-GFX10-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
+ ; GISEL-GFX10-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_bfloat
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
index 74d306b2a77dc..330fed2558778 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -735,40 +735,43 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_float(float in
define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inreg %a, half %b) {
; GISEL-GFX11-TRUE16-LABEL: name: amdgpu_cs_chain_preserve_cc_half
- ; GISEL-GFX11-TRUE16: bb.0 (%ir-block.0):
+ ; GISEL-GFX11-TRUE16: bb.1 (%ir-block.0):
; GISEL-GFX11-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-TRUE16-NEXT: {{ $}}
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]]
- ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY1]], 0, [[COPY2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GISEL-GFX11-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 killed [[COPY3]], killed [[V_ADD_F16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY]]
+ ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY4]], [[V_ADD_F16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into `ptr poison`)
; GISEL-GFX11-TRUE16-NEXT: S_ENDPGM 0
;
; GISEL-GFX11-FAKE16-LABEL: name: amdgpu_cs_chain_preserve_cc_half
- ; GISEL-GFX11-FAKE16: bb.0 (%ir-block.0):
+ ; GISEL-GFX11-FAKE16: bb.1 (%ir-block.0):
; GISEL-GFX11-FAKE16-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-FAKE16-NEXT: {{ $}}
- ; GISEL-GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; GISEL-GFX11-FAKE16-NEXT: [[V_ADD_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GISEL-GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; GISEL-GFX11-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GISEL-GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; GISEL-GFX11-FAKE16-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; GISEL-GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GISEL-GFX11-FAKE16-NEXT: [[V_ADD_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_fake16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX11-FAKE16-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; GISEL-GFX11-FAKE16-NEXT: FLAT_STORE_SHORT [[COPY3]], [[V_ADD_F16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into `ptr poison`)
; GISEL-GFX11-FAKE16-NEXT: S_ENDPGM 0
;
; GISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_half
- ; GISEL-GFX10: bb.0 (%ir-block.0):
+ ; GISEL-GFX10: bb.1 (%ir-block.0):
; GISEL-GFX10-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX10-NEXT: {{ $}}
- ; GISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; GISEL-GFX10-NEXT: [[V_ADD_F16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; GISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; GISEL-GFX10-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; GISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GISEL-GFX10-NEXT: [[V_ADD_F16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX10-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; GISEL-GFX10-NEXT: FLAT_STORE_SHORT [[COPY3]], [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into `ptr poison`)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
; DAGISEL-GFX11-WF32-TRUE16-LABEL: name: amdgpu_cs_chain_preserve_cc_half
diff --git a/llvm/test/CodeGen/AMDGPU/literal64.ll b/llvm/test/CodeGen/AMDGPU/literal64.ll
index bc66cb877e21a..385fa56b6c9a6 100644
--- a/llvm/test/CodeGen/AMDGPU/literal64.ll
+++ b/llvm/test/CodeGen/AMDGPU/literal64.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu13.10 < %s | FileCheck --check-prefixes=GFX13,GFX13-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu13.10 < %s | FileCheck --check-prefixes=GFX13,GFX13-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu13.10 < %s | FileCheck --check-prefixes=GFX13,GFX13-GISEL %s
define amdgpu_ps i64 @s_add_u64(i64 inreg %a) {
; GFX1250-LABEL: s_add_u64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
index 316f19b0c6e23..f6e87ef2bafe1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -mtriple=amdgpu9.50 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-SDAG %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.50 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu9.50 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
; RUN: llc -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-TRUE16,GFX1250-SDG-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-FAKE16,GFX1250-SDG-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-TRUE16,GFX1250-GISEL-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-FAKE16,GFX1250-GISEL-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-TRUE16,GFX1250-GISEL-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-FAKE16,GFX1250-GISEL-FAKE16 %s
declare i32 @llvm.amdgcn.bitop3.i32(i32, i32, i32, i32)
declare i16 @llvm.amdgcn.bitop3.i16(i16, i16, i16, i32)
@@ -282,15 +282,15 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
; GFX950-NEXT: v_bitop3_b16 v0, s0, v0, v1 bitop3:0x12
; GFX950-NEXT: ; return to shader part epilog
;
-; GFX1250-TRUE16-LABEL: bitop3_b16_sss:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: global_wb
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
-; GFX1250-TRUE16-NEXT: ; return to shader part epilog
+; GFX1250-SDG-TRUE16-LABEL: bitop3_b16_sss:
+; GFX1250-SDG-TRUE16: ; %bb.0:
+; GFX1250-SDG-TRUE16-NEXT: global_wb
+; GFX1250-SDG-TRUE16-NEXT: v_nop
+; GFX1250-SDG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-SDG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDG-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
+; GFX1250-SDG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-FAKE16-LABEL: bitop3_b16_sss:
; GFX1250-FAKE16: ; %bb.0:
@@ -301,17 +301,36 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_bitop3_b16 v0, s0, s1, v0 bitop3:0x12
; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: bitop3_b16_sss:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
+; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i32 18)
%ret_cast = bitcast i16 %ret to half
ret half %ret_cast
}
define amdgpu_ps half @bitop3_b16_vvi(i16 %a, i16 %b) {
-; GFX950-LABEL: bitop3_b16_vvi:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_movk_i32 s0, 0x3e8
-; GFX950-NEXT: v_bitop3_b16 v0, v0, v1, s0 bitop3:0x13
-; GFX950-NEXT: ; return to shader part epilog
+; GFX950-SDAG-LABEL: bitop3_b16_vvi:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_movk_i32 s0, 0x3e8
+; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, v1, s0 bitop3:0x13
+; GFX950-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: bitop3_b16_vvi:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0x3e8
+; GFX950-GISEL-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:0x13
+; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-TRUE16-LABEL: bitop3_b16_vvi:
; GFX1250-TRUE16: ; %bb.0:
@@ -334,12 +353,19 @@ define amdgpu_ps half @bitop3_b16_vvi(i16 %a, i16 %b) {
}
define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
-; GFX950-LABEL: bitop3_b16_vii:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_movk_i32 s0, 0x7d0
-; GFX950-NEXT: v_mov_b32_e32 v1, 0x3e8
-; GFX950-NEXT: v_bitop3_b16 v0, v0, s0, v1 bitop3:0x14
-; GFX950-NEXT: ; return to shader part epilog
+; GFX950-SDAG-LABEL: bitop3_b16_vii:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_movk_i32 s0, 0x7d0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0x3e8
+; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, s0, v1 bitop3:0x14
+; GFX950-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX950-GISEL-LABEL: bitop3_b16_vii:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 0x7d0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0x3e8
+; GFX950-GISEL-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:0x14
+; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDG-TRUE16-LABEL: bitop3_b16_vii:
; GFX1250-SDG-TRUE16: ; %bb.0:
@@ -351,15 +377,15 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
; GFX1250-SDG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0x3e8 bitop3:0x14
; GFX1250-SDG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-FAKE16-LABEL: bitop3_b16_vii:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: global_wb
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_movk_i32 s0, 0x7d0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: v_bitop3_b16 v0, v0, s0, 0x3e8 bitop3:0x14
-; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-SDG-FAKE16-LABEL: bitop3_b16_vii:
+; GFX1250-SDG-FAKE16: ; %bb.0:
+; GFX1250-SDG-FAKE16-NEXT: global_wb
+; GFX1250-SDG-FAKE16-NEXT: v_nop
+; GFX1250-SDG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDG-FAKE16-NEXT: s_movk_i32 s0, 0x7d0
+; GFX1250-SDG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDG-FAKE16-NEXT: v_bitop3_b16 v0, v0, s0, 0x3e8 bitop3:0x14
+; GFX1250-SDG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: bitop3_b16_vii:
; GFX1250-GISEL-TRUE16: ; %bb.0:
@@ -370,6 +396,16 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, 0x7d0, v0.h bitop3:0x14
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: bitop3_b16_vii:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e8
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, 0x7d0, v1 bitop3:0x14
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
%ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 2000, i16 1000, i32 20)
%ret_cast = bitcast i16 %ret to half
ret half %ret_cast
@@ -377,40 +413,71 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
; FIXME: Constant fold this
define amdgpu_ps half @bitop3_b16_iii() {
-; GFX950-LABEL: bitop3_b16_iii:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_movk_i32 s0, 0xbb8
-; GFX950-NEXT: v_mov_b32_e32 v0, 0x7d0
-; GFX950-NEXT: v_mov_b32_e32 v1, 0x3e8
-; GFX950-NEXT: v_bitop3_b16 v0, s0, v0, v1 bitop3:0x15
-; GFX950-NEXT: ; return to shader part epilog
+; GFX950-SDAG-LABEL: bitop3_b16_iii:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_movk_i32 s0, 0xbb8
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0x7d0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0x3e8
+; GFX950-SDAG-NEXT: v_bitop3_b16 v0, s0, v0, v1 bitop3:0x15
+; GFX950-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-TRUE16-LABEL: bitop3_b16_iii:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: global_wb
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x7d0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xbb8
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.h, v0.l, 0x3e8 bitop3:0x15
-; GFX1250-TRUE16-NEXT: ; return to shader part epilog
+; GFX950-GISEL-LABEL: bitop3_b16_iii:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0xbb8
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 0x7d0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0x3e8
+; GFX950-GISEL-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:0x15
+; GFX950-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-FAKE16-LABEL: bitop3_b16_iii:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: global_wb
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, 0x3e8
-; GFX1250-FAKE16-NEXT: s_movk_i32 s0, 0xbb8
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: v_bitop3_b16 v0, s0, 0x7d0, v0 bitop3:0x15
-; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-SDG-TRUE16-LABEL: bitop3_b16_iii:
+; GFX1250-SDG-TRUE16: ; %bb.0:
+; GFX1250-SDG-TRUE16-NEXT: global_wb
+; GFX1250-SDG-TRUE16-NEXT: v_nop
+; GFX1250-SDG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x7d0
+; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xbb8
+; GFX1250-SDG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.h, v0.l, 0x3e8 bitop3:0x15
+; GFX1250-SDG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDG-FAKE16-LABEL: bitop3_b16_iii:
+; GFX1250-SDG-FAKE16: ; %bb.0:
+; GFX1250-SDG-FAKE16-NEXT: global_wb
+; GFX1250-SDG-FAKE16-NEXT: v_nop
+; GFX1250-SDG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDG-FAKE16-NEXT: v_mov_b32_e32 v0, 0x3e8
+; GFX1250-SDG-FAKE16-NEXT: s_movk_i32 s0, 0xbb8
+; GFX1250-SDG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDG-FAKE16-NEXT: v_bitop3_b16 v0, s0, 0x7d0, v0 bitop3:0x15
+; GFX1250-SDG-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: bitop3_b16_iii:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x7d0
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3e8
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, 0xbb8, v0.l, v0.h bitop3:0x15
+; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: bitop3_b16_iii:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7d0
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e8
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, 0xbb8, v0, v1 bitop3:0x15
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
%ret = call i16 @llvm.amdgcn.bitop3.i16(i16 3000, i16 2000, i16 1000, i32 21)
%ret_cast = bitcast i16 %ret to half
ret half %ret_cast
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
-; GFX1250-GISEL-FAKE16: {{.*}}
-; GFX1250-SDG-FAKE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
index c23fbc151db25..b95212b16e5ac 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI-GISEL %s
+; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI-GISEL %s
declare half @llvm.fabs.f16(half %a)
declare i1 @llvm.amdgcn.class.f16(half %a, i32 %b)
@@ -65,21 +65,37 @@ entry:
}
define amdgpu_kernel void @class_f16_fabs(
-; GCN-LABEL: class_f16_fabs:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s4, s[8:9], 0x4c
-; GCN-NEXT: s_load_dword s5, s[8:9], 0x28
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: s_mov_b32 s3, 0x1100f000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_cmp_class_f16_e64 s[4:5], |s5|, v0
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s4, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; VI-SDAG-LABEL: class_f16_fabs:
+; VI-SDAG: ; %bb.0: ; %entry
+; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x4c
+; VI-SDAG-NEXT: s_load_dword s5, s[8:9], 0x28
+; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], |s5|, v0
+; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: class_f16_fabs:
+; VI-GISEL: ; %bb.0: ; %entry
+; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x4c
+; VI-GISEL-NEXT: s_load_dword s4, s[8:9], 0x28
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], |s4|, v0
+; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
[8 x i32],
half %a.val,
@@ -94,21 +110,37 @@ entry:
}
define amdgpu_kernel void @class_f16_fneg(
-; GCN-LABEL: class_f16_fneg:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s4, s[8:9], 0x4c
-; GCN-NEXT: s_load_dword s5, s[8:9], 0x28
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: s_mov_b32 s3, 0x1100f000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_cmp_class_f16_e64 s[4:5], -s5, v0
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s4, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; VI-SDAG-LABEL: class_f16_fneg:
+; VI-SDAG: ; %bb.0: ; %entry
+; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x4c
+; VI-SDAG-NEXT: s_load_dword s5, s[8:9], 0x28
+; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], -s5, v0
+; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: class_f16_fneg:
+; VI-GISEL: ; %bb.0: ; %entry
+; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x28
+; VI-GISEL-NEXT: s_load_dword s4, s[8:9], 0x4c
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_max_f16_e64 v0, -s3, -s3
+; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, s4
+; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
[8 x i32],
half %a.val,
@@ -123,21 +155,37 @@ entry:
}
define amdgpu_kernel void @class_f16_fabs_fneg(
-; GCN-LABEL: class_f16_fabs_fneg:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s4, s[8:9], 0x4c
-; GCN-NEXT: s_load_dword s5, s[8:9], 0x28
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: s_mov_b32 s3, 0x1100f000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_cmp_class_f16_e64 s[4:5], -|s5|, v0
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s4, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; VI-SDAG-LABEL: class_f16_fabs_fneg:
+; VI-SDAG: ; %bb.0: ; %entry
+; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x4c
+; VI-SDAG-NEXT: s_load_dword s5, s[8:9], 0x28
+; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], -|s5|, v0
+; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: class_f16_fabs_fneg:
+; VI-GISEL: ; %bb.0: ; %entry
+; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x28
+; VI-GISEL-NEXT: s_load_dword s4, s[8:9], 0x4c
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_max_f16_e64 v0, -|s3|, -|s3|
+; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, s4
+; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
[8 x i32],
half %a.val,
@@ -153,19 +201,33 @@ entry:
}
define amdgpu_kernel void @class_f16_1(
-; GCN-LABEL: class_f16_1:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s4, s[8:9], 0x8
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: s_mov_b32 s3, 0x1100f000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 1
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s4, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; VI-SDAG-LABEL: class_f16_1:
+; VI-SDAG: ; %bb.0: ; %entry
+; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
+; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 1
+; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: class_f16_1:
+; VI-GISEL: ; %bb.0: ; %entry
+; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], s3, 1
+; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
half %a.val) {
entry:
@@ -176,19 +238,33 @@ entry:
}
define amdgpu_kernel void @class_f16_64(
-; GCN-LABEL: class_f16_64:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s4, s[8:9], 0x8
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: s_mov_b32 s3, 0x1100f000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 64
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s4, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; VI-SDAG-LABEL: class_f16_64:
+; VI-SDAG: ; %bb.0: ; %entry
+; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
+; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 64
+; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: class_f16_64:
+; VI-GISEL: ; %bb.0: ; %entry
+; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], s3, 64
+; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
half %a.val) {
entry:
@@ -199,20 +275,35 @@ entry:
}
define amdgpu_kernel void @class_f16_full_mask(
-; GCN-LABEL: class_f16_full_mask:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s4, s[8:9], 0x8
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: v_mov_b32_e32 v0, 0x3ff
-; GCN-NEXT: s_mov_b32 s3, 0x1100f000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
-; GCN-NEXT: s_and_b64 s[4:5], vcc, exec
-; GCN-NEXT: s_cselect_b32 s4, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; VI-SDAG-LABEL: class_f16_full_mask:
+; VI-SDAG: ; %bb.0: ; %entry
+; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
+; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x3ff
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: class_f16_full_mask:
+; VI-GISEL: ; %bb.0: ; %entry
+; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x3ff
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cmp_class_f16_e32 vcc, s3, v0
+; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
half %a.val) {
entry:
@@ -223,20 +314,35 @@ entry:
}
define amdgpu_kernel void @class_f16_nine_bit_mask(
-; GCN-LABEL: class_f16_nine_bit_mask:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s4, s[8:9], 0x8
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: v_mov_b32_e32 v0, 0x1ff
-; GCN-NEXT: s_mov_b32 s3, 0x1100f000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
-; GCN-NEXT: s_and_b64 s[4:5], vcc, exec
-; GCN-NEXT: s_cselect_b32 s4, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; VI-SDAG-LABEL: class_f16_nine_bit_mask:
+; VI-SDAG: ; %bb.0: ; %entry
+; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
+; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x1ff
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: class_f16_nine_bit_mask:
+; VI-GISEL: ; %bb.0: ; %entry
+; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x1ff
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cmp_class_f16_e32 vcc, s3, v0
+; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
half %a.val) {
entry:
@@ -245,3 +351,5 @@ entry:
store i32 %r.val.sext, ptr addrspace(1) %r
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll
index f2b613bbb40c6..3f09037dd4041 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
declare void @llvm.amdgcn.cluster.load.async.to.lds.b8(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %offset, i32 %cpol, i32 %mask)
declare void @llvm.amdgcn.cluster.load.async.to.lds.b32(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %offset, i32 %cpol, i32 %mask)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
index 24960a42af0d0..96d12677cdc88 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
@@ -299,9 +299,9 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f16(half %a, i32 %sr, i32 %old, i32 0)
store i32 %cvt, ptr addrspace(1) %out
@@ -360,9 +360,9 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:1
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f16(half %a, i32 %sr, i32 %old, i32 1)
store i32 %cvt, ptr addrspace(1) %out
@@ -405,9 +405,9 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:2
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f16(half %a, i32 %sr, i32 %old, i32 2)
store i32 %cvt, ptr addrspace(1) %out
@@ -450,9 +450,9 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:3
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f16(half %a, i32 %sr, i32 %old, i32 3)
store i32 %cvt, ptr addrspace(1) %out
@@ -545,9 +545,9 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f16(half %a, i32 %sr, i32 %old, i32 0)
store i32 %cvt, ptr addrspace(1) %out
@@ -606,9 +606,9 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:1
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f16(half %a, i32 %sr, i32 %old, i32 1)
store i32 %cvt, ptr addrspace(1) %out
@@ -651,9 +651,9 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:2
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f16(half %a, i32 %sr, i32 %old, i32 2)
store i32 %cvt, ptr addrspace(1) %out
@@ -696,9 +696,9 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:3
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f16(half %a, i32 %sr, i32 %old, i32 3)
store i32 %cvt, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
index 1b8ba413172cf..edd9543d861cc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=ASM-SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=ASM-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=ASM-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=ASM-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=ASM-GISEL-FAKE16 %s
; Test that we can use v0 for temporaries in the if.then block.
define i32 @dead_i32(i1 %cond, i32 %x, ptr addrspace(1) %ptr1) #0 {
@@ -771,44 +771,47 @@ define %non_trivial_types @dead_non_trivial(i1 %cond, %non_trivial_types %x, ptr
; ASM-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; ASM-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; ASM-GISEL-FAKE16-NEXT: s_clause 0x15
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v32, off, s32 offset:80
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:76
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v33, off, s32 offset:72
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v34, off, s32 offset:68
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v35, off, s32 offset:64
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v36, off, s32 offset:60
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v37, off, s32 offset:56
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v38, off, s32 offset:52
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v39, off, s32 offset:48
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v48, off, s32 offset:44
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v49, off, s32 offset:40
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v33, off, s32
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v34, off, s32 offset:4
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v35, off, s32 offset:8
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v36, off, s32 offset:12
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v37, off, s32 offset:16
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v38, off, s32 offset:20
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v39, off, s32 offset:24
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v48, off, s32 offset:28
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v49, off, s32 offset:32
; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v50, off, s32 offset:36
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v51, off, s32 offset:32
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v52, off, s32 offset:28
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v53, off, s32 offset:24
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:20
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v55, off, s32 offset:16
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v64, off, s32 offset:12
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v65, off, s32 offset:8
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v66, off, s32 offset:4
-; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v67, off, s32
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v51, off, s32 offset:40
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v52, off, s32 offset:44
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v53, off, s32 offset:48
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:52
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v55, off, s32 offset:56
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v64, off, s32 offset:60
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v65, off, s32 offset:64
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v66, off, s32 offset:68
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v67, off, s32 offset:72
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:76
+; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v32, off, s32 offset:80
; ASM-GISEL-FAKE16-NEXT: scratch_load_b32 v68, off, s32 offset:84
; ASM-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 1, v1
; ASM-GISEL-FAKE16-NEXT: s_mov_b32 s0, exec_lo
; ASM-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; ASM-GISEL-FAKE16-NEXT: v_cmpx_eq_u32_e32 1, v1
+; ASM-GISEL-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; ASM-GISEL-FAKE16-NEXT: s_cbranch_execz .LBB3_2
; ASM-GISEL-FAKE16-NEXT: ; %bb.1: ; %if.then
+; ASM-GISEL-FAKE16-NEXT: s_mov_b32 s1, 0
+; ASM-GISEL-FAKE16-NEXT: s_movk_i32 s2, 0x3e00
; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; ASM-GISEL-FAKE16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_add_nc_u32 v1, 15, v68
-; ASM-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0x3e00
+; ASM-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; ASM-GISEL-FAKE16-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_add_nc_u32 v1, 15, v68
+; ASM-GISEL-FAKE16-NEXT: v_mov_b32_e32 v8, s1
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr2
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr3
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr5
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr6
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr7
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11_vgpr12_vgpr13
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr14_vgpr15
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr14_vgpr15_vgpr16_vgpr17
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr18
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr19
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr20
@@ -822,42 +825,42 @@ define %non_trivial_types @dead_non_trivial(i1 %cond, %non_trivial_types %x, ptr
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr28
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr29
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr30
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr67
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr66
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr65
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr64
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr55
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr54
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr53
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr52
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr51
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr50
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr49
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr48
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr39
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr38
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr37
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr36
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr35
-; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr34
; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr33
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr34
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr35
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr36
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr37
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr38
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr39
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr48
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr49
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr50
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr51
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr52
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr53
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr54
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr55
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr64
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr65
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr66
+; ASM-GISEL-FAKE16-NEXT: ; implicit-def: $vgpr67
; ASM-GISEL-FAKE16-NEXT: global_store_b32 v[31:32], v1, off
; ASM-GISEL-FAKE16-NEXT: .LBB3_2: ; %if.end
; ASM-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; ASM-GISEL-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; ASM-GISEL-FAKE16-NEXT: s_clause 0x16
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b16 v0, v4, off offset:4
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b16 v0, v3, off offset:2
; ASM-GISEL-FAKE16-NEXT: scratch_store_b8 v0, v2, off
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b16 v0, v3, off offset:2
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b16 v0, v4, off offset:4
; ASM-GISEL-FAKE16-NEXT: scratch_store_b16 v0, v5, off offset:6
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v6, off offset:8
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v7, off offset:12
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v8, off offset:16
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v13, off offset:48
; ASM-GISEL-FAKE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v13, off offset:48
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:64
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v18, off offset:80
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v19, off offset:84
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:64
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v20, off offset:88
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v21, off offset:92
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v22, off offset:96
@@ -869,27 +872,44 @@ define %non_trivial_types @dead_non_trivial(i1 %cond, %non_trivial_types %x, ptr
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v28, off offset:120
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v29, off offset:124
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v30, off offset:128
-; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x1
-; ASM-GISEL-FAKE16-NEXT: s_clause 0x12
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v67, off offset:132
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v66, off offset:136
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v65, off offset:140
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v64, off offset:144
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v55, off offset:148
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v54, off offset:152
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v53, off offset:156
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v52, off offset:160
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v51, off offset:164
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x15
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v33, off offset:132
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x14
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v34, off offset:136
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x13
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v35, off offset:140
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x12
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v36, off offset:144
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x11
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v37, off offset:148
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x10
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v38, off offset:152
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0xf
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v39, off offset:156
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0xe
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v48, off offset:160
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0xd
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v49, off offset:164
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0xc
; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v50, off offset:168
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v49, off offset:172
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v48, off offset:176
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v39, off offset:180
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v38, off offset:184
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v37, off offset:188
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v36, off offset:192
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v35, off offset:196
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v34, off offset:200
-; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v33, off offset:204
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0xb
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v51, off offset:172
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0xa
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v52, off offset:176
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x9
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v53, off offset:180
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x8
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v54, off offset:184
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x7
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v55, off offset:188
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x6
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v64, off offset:192
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x5
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v65, off offset:196
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x4
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v66, off offset:200
+; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x3
+; ASM-GISEL-FAKE16-NEXT: scratch_store_b32 v0, v67, off offset:204
; ASM-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
; ASM-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
; ASM-GISEL-LABEL: dead_non_trivial:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
index 8a4c2297a8305..a560faf6182df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "load" --filter-out "store" --filter-out "wait" --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI-GISEL %s
declare half @llvm.amdgcn.div.fixup.f16(half %a, half %b, half %c)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
index c6176017e7010..0122b70985c86 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12 %s
define float @v_exp2_f32(float %src) {
; GCN-LABEL: v_exp2_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
index e163b7bfeb6ed..3f37b9b7ef0e2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX11 %s
; RUN: llc -mtriple=amdgpu10.10 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX10 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
declare i32 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i32 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
@@ -1569,12 +1569,13 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: s_lshr_b32 s3, s2, 16
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GISEL-GFX11-NEXT: v_cmp_eq_f16_e64 s2, s2, |s3|
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_oeq_with_fabs:
@@ -1582,12 +1583,12 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: s_lshr_b32 s3, s2, 16
; GISEL-GFX10-NEXT: v_cmp_eq_f16_e64 s2, s2, |s3|
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%temp = call half @llvm.fabs.f16(half %a)
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half %temp, i32 1)
@@ -1628,12 +1629,13 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: s_lshr_b32 s3, s2, 16
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GISEL-GFX11-NEXT: v_cmp_eq_f16_e64 s2, |s2|, |s3|
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
@@ -1641,12 +1643,12 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: s_lshr_b32 s3, s2, 16
; GISEL-GFX10-NEXT: v_cmp_eq_f16_e64 s2, |s2|, |s3|
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%temp = call half @llvm.fabs.f16(half %a)
%src_input = call half @llvm.fabs.f16(half %src)
@@ -1666,10 +1668,18 @@ define amdgpu_kernel void @v_fcmp_f16(ptr addrspace(1) %out, half %src) {
;
; GISEL-GFX11-LABEL: v_fcmp_f16:
; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: global_store_b32 v0, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16:
; GISEL-GFX10: ; %bb.0:
+; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT: global_store_dword v0, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 -1)
store i32 %result, ptr addrspace(1) %out
@@ -1707,11 +1717,12 @@ define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_eq_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_oeq:
@@ -1719,11 +1730,11 @@ define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_eq_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 1)
store i32 %result, ptr addrspace(1) %out
@@ -1761,11 +1772,12 @@ define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_one:
@@ -1773,11 +1785,11 @@ define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 6)
store i32 %result, ptr addrspace(1) %out
@@ -1815,11 +1827,12 @@ define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_lt_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_ogt:
@@ -1827,11 +1840,11 @@ define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_lt_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 2)
store i32 %result, ptr addrspace(1) %out
@@ -1869,11 +1882,12 @@ define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_le_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_oge:
@@ -1881,11 +1895,11 @@ define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_le_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 3)
store i32 %result, ptr addrspace(1) %out
@@ -1923,11 +1937,12 @@ define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_gt_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_olt:
@@ -1935,11 +1950,11 @@ define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_gt_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 4)
store i32 %result, ptr addrspace(1) %out
@@ -1977,11 +1992,12 @@ define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_ge_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_ole:
@@ -1989,11 +2005,11 @@ define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_ge_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 5)
store i32 %result, ptr addrspace(1) %out
@@ -2031,11 +2047,12 @@ define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_nlg_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_ueq:
@@ -2043,11 +2060,11 @@ define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_nlg_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 9)
store i32 %result, ptr addrspace(1) %out
@@ -2085,11 +2102,12 @@ define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_une:
@@ -2097,11 +2115,11 @@ define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 14)
store i32 %result, ptr addrspace(1) %out
@@ -2139,11 +2157,12 @@ define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_nge_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_ugt:
@@ -2151,11 +2170,11 @@ define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_nge_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 10)
store i32 %result, ptr addrspace(1) %out
@@ -2193,11 +2212,12 @@ define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_ngt_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_uge:
@@ -2205,11 +2225,11 @@ define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_ngt_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 11)
store i32 %result, ptr addrspace(1) %out
@@ -2247,11 +2267,12 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_nle_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_ult:
@@ -2259,11 +2280,11 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_nle_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 12)
store i32 %result, ptr addrspace(1) %out
@@ -2300,11 +2321,12 @@ define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_o_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_o:
@@ -2312,11 +2334,11 @@ define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_o_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 7)
store i32 %result, ptr addrspace(1) %out
@@ -2353,11 +2375,12 @@ define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_u_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_uo:
@@ -2365,11 +2388,11 @@ define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_u_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 8)
store i32 %result, ptr addrspace(1) %out
@@ -2406,11 +2429,12 @@ define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
; GISEL-GFX11-NEXT: s_clause 0x1
; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX11-NEXT: v_cmp_nlt_f16_e64 s2, 0x5640, s2
; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GISEL-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-GFX10-LABEL: v_fcmp_f16_ule:
@@ -2418,11 +2442,11 @@ define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
; GISEL-GFX10-NEXT: s_clause 0x1
; GISEL-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX10-NEXT: v_cmp_nlt_f16_e64 s2, 0x5640, s2
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GISEL-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 13)
store i32 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 86f22fa12fa06..91a7e4937bdee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.00 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI-GISEL %s
declare i64 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i64 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
@@ -1862,19 +1862,33 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_oeq_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], v0, |s3|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
; VI-SDAG: ; %bb.0:
@@ -1897,13 +1911,13 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: s_lshr_b32 s3, s2, 16
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], v0, |s3|
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%temp = call half @llvm.fabs.f16(half %a)
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half %temp, i32 1)
@@ -1929,19 +1943,33 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |v0|, |s3|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
; VI-SDAG: ; %bb.0:
@@ -1964,13 +1992,13 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: s_lshr_b32 s3, s2, 16
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |v0|, |s3|
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%temp = call half @llvm.fabs.f16(half %a)
%src_input = call half @llvm.fabs.f16(half %src)
@@ -1980,13 +2008,29 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
}
define amdgpu_kernel void @v_fcmp_f16(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: global_store_b64 v0, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16:
; VI-SDAG: ; %bb.0:
@@ -1994,6 +2038,11 @@ define amdgpu_kernel void @v_fcmp_f16(ptr addrspace(1) %out, half %src) {
;
; VI-GISEL-LABEL: v_fcmp_f16:
; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 -1)
store i64 %result, ptr addrspace(1) %out
@@ -2050,11 +2099,11 @@ define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 1)
store i64 %result, ptr addrspace(1) %out
@@ -2111,11 +2160,11 @@ define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 6)
store i64 %result, ptr addrspace(1) %out
@@ -2172,11 +2221,11 @@ define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_gt_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 2)
store i64 %result, ptr addrspace(1) %out
@@ -2233,11 +2282,11 @@ define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_ge_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 3)
store i64 %result, ptr addrspace(1) %out
@@ -2294,11 +2343,11 @@ define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 4)
store i64 %result, ptr addrspace(1) %out
@@ -2355,11 +2404,11 @@ define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_le_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 5)
store i64 %result, ptr addrspace(1) %out
@@ -2416,11 +2465,11 @@ define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_nlg_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 9)
store i64 %result, ptr addrspace(1) %out
@@ -2477,11 +2526,11 @@ define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 14)
store i64 %result, ptr addrspace(1) %out
@@ -2538,11 +2587,11 @@ define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_nle_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 10)
store i64 %result, ptr addrspace(1) %out
@@ -2599,11 +2648,11 @@ define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_nlt_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 11)
store i64 %result, ptr addrspace(1) %out
@@ -2660,11 +2709,11 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_nge_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 12)
store i64 %result, ptr addrspace(1) %out
@@ -2720,11 +2769,11 @@ define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_o_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 7)
store i64 %result, ptr addrspace(1) %out
@@ -2780,11 +2829,11 @@ define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_u_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 8)
store i64 %result, ptr addrspace(1) %out
@@ -2840,11 +2889,11 @@ define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_ngt_f16_e64 s[2:3], s2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; VI-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 13)
store i64 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
index c2217b6e18224..e4704a81cfa97 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefix=GCN %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefix=GCN %s
declare void @llvm.amdgcn.flat.prefetch(ptr %ptr, i32 %col)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
index f9d186a42adc1..3dbf26f79cf47 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -denormal-fp-math-f32=ieee < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 -denormal-fp-math-f32=ieee < %s | FileCheck -check-prefixes=GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 -denormal-fp-math-f32=ieee < %s | FileCheck -check-prefixes=GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -denormal-fp-math-f32=ieee < %s | FileCheck -check-prefixes=GFX9-GISEL %s
declare half @llvm.amdgcn.fmad.ftz.f16(half %a, half %b, half %c)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.f16.ll
index 8a821daab4cfe..dfdd5a54bd884 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI-GISEL %s
declare i16 @llvm.amdgcn.frexp.exp.i16.f16(half %a)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
index 54e73e2bc95cb..4fc98cb7ff6d5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
declare void @llvm.amdgcn.global.load.async.to.lds.b8(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %offset, i32 %cpol)
declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %offset, i32 %cpol)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
index 5e274d535531b..b05ee073aa9ce 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefix=GCN %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefix=GCN %s
declare void @llvm.amdgcn.global.prefetch(ptr addrspace(1) %ptr, i32 %col)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll
index 2ee998c26de8b..13c83ff62be93 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
declare void @llvm.amdgcn.global.store.async.from.lds.b8(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %offset, i32 %cpol)
declare void @llvm.amdgcn.global.store.async.from.lds.b32(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %offset, i32 %cpol)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
index 334685acda0f2..556fb9647309c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX10GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel < %s | FileCheck -check-prefixes=GFX10GISEL %s
; TODO: global-isel produces more code - there will need to be some more combines in the postregbankselectcombine phase
; Depends on some other changes to pass this test - those are in review separately
@@ -13,6 +13,8 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -31,7 +33,8 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -57,15 +60,11 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_3d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v15, v8
-; GFX10GISEL-NEXT: v_mov_b32_e32 v13, v5
-; GFX10GISEL-NEXT: v_mov_b32_e32 v12, v4
-; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v1
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX10GISEL-NEXT: v_perm_b32 v14, v7, v6, 0x5040100
-; GFX10GISEL-NEXT: image_sample_d v[0:3], v[8:15], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -82,6 +81,8 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX10GISEL-LABEL: sample_c_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
; GFX10GISEL-NEXT: image_sample_c_d v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -100,7 +101,8 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX10GISEL-LABEL: sample_c_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v5, v6, v5, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX10GISEL-NEXT: image_sample_c_d v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -119,7 +121,8 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX10GISEL-NEXT: image_sample_d_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -143,13 +146,11 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v6
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v7, v1
-; GFX10GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX10GISEL-NEXT: v_perm_b32 v10, v5, v4, 0x5040100
-; GFX10GISEL-NEXT: image_sample_d_cl v[0:3], v[6:11], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: image_sample_d_cl v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -167,7 +168,8 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX10GISEL-NEXT: image_sample_c_d_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -192,14 +194,11 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v13, v7
-; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v4
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v1
-; GFX10GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX10GISEL-NEXT: v_perm_b32 v12, v6, v5, 0x5040100
-; GFX10GISEL-NEXT: image_sample_c_d_cl v[0:3], v[7:13], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX10GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10GISEL-NEXT: v_lshl_or_b32 v6, s0, 16, v7
+; GFX10GISEL-NEXT: image_sample_c_d_cl v[0:3], v[0:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -216,6 +215,8 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX10GISEL-LABEL: sample_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10GISEL-NEXT: image_sample_cd v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -234,7 +235,8 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX10GISEL-LABEL: sample_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
; GFX10GISEL-NEXT: image_sample_cd v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -252,6 +254,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_c_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
; GFX10GISEL-NEXT: image_sample_c_cd v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -270,7 +274,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_c_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v5, v6, v5, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX10GISEL-NEXT: image_sample_c_cd v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -289,7 +294,8 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX10GISEL-LABEL: sample_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX10GISEL-NEXT: image_sample_cd_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -313,13 +319,11 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX10GISEL-LABEL: sample_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v6
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v7, v1
-; GFX10GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX10GISEL-NEXT: v_perm_b32 v10, v5, v4, 0x5040100
-; GFX10GISEL-NEXT: image_sample_cd_cl v[0:3], v[6:11], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: image_sample_cd_cl v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -337,7 +341,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX10GISEL-NEXT: image_sample_c_cd_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -362,14 +367,11 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v13, v7
-; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v4
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v1
-; GFX10GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX10GISEL-NEXT: v_perm_b32 v12, v6, v5, 0x5040100
-; GFX10GISEL-NEXT: image_sample_c_cd_cl v[0:3], v[7:13], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX10GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10GISEL-NEXT: v_lshl_or_b32 v6, s0, 16, v7
+; GFX10GISEL-NEXT: image_sample_c_cd_cl v[0:3], v[0:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -394,15 +396,11 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_o_2darray_V1:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v15, v8
-; GFX10GISEL-NEXT: v_mov_b32_e32 v13, v5
-; GFX10GISEL-NEXT: v_mov_b32_e32 v12, v4
-; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v1
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX10GISEL-NEXT: v_perm_b32 v14, v7, v6, 0x5040100
-; GFX10GISEL-NEXT: image_sample_c_d_o v0, v[8:15], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: image_sample_c_d_o v0, v[0:7], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -427,15 +425,11 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_c_d_o_2darray_V2:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v15, v8
-; GFX10GISEL-NEXT: v_mov_b32_e32 v13, v5
-; GFX10GISEL-NEXT: v_mov_b32_e32 v12, v4
-; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v1
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX10GISEL-NEXT: v_perm_b32 v14, v7, v6, 0x5040100
-; GFX10GISEL-NEXT: image_sample_c_d_o v[0:1], v[8:15], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: image_sample_c_d_o v[0:1], v[0:7], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -474,6 +468,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_1d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -493,9 +491,11 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_2d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -516,10 +516,14 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_3d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_3d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v2
-; GFX10GISEL-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v4, v4, v9, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v2
+; GFX10GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v3
+; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v1, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v9
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v10
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v5
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[2:8], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -537,6 +541,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10GISEL-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -556,9 +564,11 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10GISEL-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10GISEL-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -575,6 +585,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_cl_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10GISEL-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -594,9 +608,11 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_cl_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10GISEL-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -613,6 +629,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_cl_1d(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10GISEL-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -636,8 +656,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_cl_2d(<8 x i32> inreg %rsrc,
; GFX10GISEL: ; %bb.0: ; %main_body
; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_perm_b32 v4, v4, v3, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v3, v8, v1, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v8, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v1
; GFX10GISEL-NEXT: image_sample_c_d_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -655,6 +677,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_1d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10GISEL-NEXT: image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -674,9 +700,11 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_2d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10GISEL-NEXT: image_sample_cd_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: image_sample_cd_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -693,6 +721,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10GISEL-NEXT: image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -712,9 +744,11 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
-; GFX10GISEL-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10GISEL-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -731,6 +765,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_cl_1d(<8 x i32> inreg %rsrc, <
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10GISEL-NEXT: image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -750,9 +788,11 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_cl_2d(<8 x i32> inreg %rsrc, <
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX10GISEL-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -769,6 +809,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_cl_1d(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10GISEL-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -792,8 +836,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_cl_2d(<8 x i32> inreg %rsrc,
; GFX10GISEL: ; %bb.0: ; %main_body
; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_perm_b32 v4, v4, v3, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v3, v8, v1, 0x5040100
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v8, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v1
; GFX10GISEL-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -817,12 +863,14 @@ define amdgpu_ps float @sample_g16_noa16_c_d_o_2darray_V1(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_o_2darray_V1:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v4, v9, v10, 0x5040100
+; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v5, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v10, 16, v0
; GFX10GISEL-NEXT: image_sample_c_d_o_g16 v0, v[2:8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -846,12 +894,14 @@ define amdgpu_ps <2 x float> @sample_g16_noa16_c_d_o_2darray_V2(<8 x i32> inreg
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_o_2darray_V2:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v4, v9, v10, 0x5040100
+; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v5, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v10, 16, v0
; GFX10GISEL-NEXT: image_sample_c_d_o_g16 v[0:1], v[2:8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -891,6 +941,12 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_1d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -911,10 +967,13 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_2d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v3, v3, v2, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
-; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v5, 16, v4
+; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
@@ -937,13 +996,19 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_3d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v12, v8
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v5
-; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
-; GFX10GISEL-NEXT: v_perm_b32 v11, v7, v6, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v9, v4, v3, 0x5040100
-; GFX10GISEL-NEXT: v_perm_b32 v7, v1, v0, 0x5040100
-; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[7:12], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v5
+; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v6
+; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v8
+; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v2
+; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v7, 16, v5
+; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
index abe75641a2573..b913a41094b48 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX10_11,GFX10_11-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX10_11,GFX10_11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX10_11,GFX10_11-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10_11,GFX10_11-SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10_11,GFX10_11-SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-GISEL,GFX12PLUS-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-GISEL,GFX12PLUS-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-GISEL,GFX12PLUS-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu13.10 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu13.10 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu13.10 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-GISEL,GFX12PLUS-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu13.10 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-GISEL,GFX12PLUS-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu13.10 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12PLUS-GISEL,GFX12PLUS-GISEL-FAKE16 %s
define amdgpu_ps void @sample_1d_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
; GFX10_11-LABEL: sample_1d_nortn:
@@ -437,15 +437,38 @@ main_body:
}
define amdgpu_ps void @sample_d_1d_g16_nortn(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
-; GFX10_11-LABEL: sample_d_1d_g16_nortn:
-; GFX10_11: ; %bb.0: ; %main_body
-; GFX10_11-NEXT: image_sample_d_g16 off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10_11-NEXT: s_endpgm
+; GFX10_11-SDAG-LABEL: sample_d_1d_g16_nortn:
+; GFX10_11-SDAG: ; %bb.0: ; %main_body
+; GFX10_11-SDAG-NEXT: image_sample_d_g16 off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10_11-SDAG-NEXT: s_endpgm
;
-; GFX12PLUS-LABEL: sample_d_1d_g16_nortn:
-; GFX12PLUS: ; %bb.0: ; %main_body
-; GFX12PLUS-NEXT: image_sample_d_g16 off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX12PLUS-NEXT: s_endpgm
+; GFX10_11-GISEL-LABEL: sample_d_1d_g16_nortn:
+; GFX10_11-GISEL: ; %bb.0: ; %main_body
+; GFX10_11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10_11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10_11-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX10_11-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10_11-GISEL-NEXT: image_sample_d_g16 off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX10_11-GISEL-NEXT: s_endpgm
+;
+; GFX12PLUS-SDAG-LABEL: sample_d_1d_g16_nortn:
+; GFX12PLUS-SDAG: ; %bb.0: ; %main_body
+; GFX12PLUS-SDAG-NEXT: image_sample_d_g16 off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12PLUS-SDAG-NEXT: s_endpgm
+;
+; GFX12PLUS-GISEL-TRUE16-LABEL: sample_d_1d_g16_nortn:
+; GFX12PLUS-GISEL-TRUE16: ; %bb.0: ; %main_body
+; GFX12PLUS-GISEL-TRUE16-NEXT: image_sample_d_g16 off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12PLUS-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX12PLUS-GISEL-FAKE16-LABEL: sample_d_1d_g16_nortn:
+; GFX12PLUS-GISEL-FAKE16: ; %bb.0: ; %main_body
+; GFX12PLUS-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12PLUS-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12PLUS-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX12PLUS-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX12PLUS-GISEL-FAKE16-NEXT: image_sample_d_g16 off, [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12PLUS-GISEL-FAKE16-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.image.sample.d.1d.nortn.f16.f32(i32 15, half %dsdh, half %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
ret void
@@ -476,6 +499,3 @@ declare void @llvm.amdgcn.image.sample.d.1d.nortn.f16.f32(i32, half, half, float
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12PLUS-GISEL-FAKE16: {{.*}}
-; GFX12PLUS-GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
index c75efd6182b15..bfac0611cd6c3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-32BANK %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-32BANK %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-32BANK %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX8-32BANK %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX8-32BANK %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX8-32BANK %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.10 < %s | FileCheck -check-prefixes=GFX8-16BANK-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.10 < %s | FileCheck -check-prefixes=GFX8-16BANK-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.10 < %s | FileCheck -check-prefixes=GFX8-16BANK-GISEL %s
define amdgpu_ps half @interp_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
; GFX9-32BANK-LABEL: interp_f16:
@@ -131,10 +131,10 @@ define amdgpu_ps half @interp_p1_m0_setup(float inreg %i, float inreg %j, i32 in
; GFX8-16BANK-GISEL-NEXT: ;;#ASMEND
; GFX8-16BANK-GISEL-NEXT: s_mov_b32 s3, m0
; GFX8-16BANK-GISEL-NEXT: s_mov_b32 m0, s2
-; GFX8-16BANK-GISEL-NEXT: v_interp_mov_f32_e32 v0, p0, attr2.y
-; GFX8-16BANK-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-16BANK-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-16BANK-GISEL-NEXT: v_interp_mov_f32_e32 v1, p0, attr2.y
; GFX8-16BANK-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
-; GFX8-16BANK-GISEL-NEXT: v_interp_p1lv_f16 v0, v1, attr2.y, v0
+; GFX8-16BANK-GISEL-NEXT: v_interp_p1lv_f16 v0, v0, attr2.y, v1
; GFX8-16BANK-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX8-16BANK-GISEL-NEXT: v_interp_p2_f16 v0, v1, attr2.y, v0
; GFX8-16BANK-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -207,10 +207,10 @@ define amdgpu_ps half @interp_p2_m0_setup(float inreg %i, float inreg %j, i32 in
; GFX8-16BANK-GISEL-LABEL: interp_p2_m0_setup:
; GFX8-16BANK-GISEL: ; %bb.0: ; %main_body
; GFX8-16BANK-GISEL-NEXT: s_mov_b32 m0, s2
-; GFX8-16BANK-GISEL-NEXT: v_interp_mov_f32_e32 v0, p0, attr2.y
-; GFX8-16BANK-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-16BANK-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-16BANK-GISEL-NEXT: v_interp_mov_f32_e32 v1, p0, attr2.y
; GFX8-16BANK-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
-; GFX8-16BANK-GISEL-NEXT: v_interp_p1lv_f16 v0, v1, attr2.y, v0
+; GFX8-16BANK-GISEL-NEXT: v_interp_p1lv_f16 v0, v0, attr2.y, v1
; GFX8-16BANK-GISEL-NEXT: ;;#ASMSTART
; GFX8-16BANK-GISEL-NEXT: s_mov_b32 m0, 0
; GFX8-16BANK-GISEL-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
index 4ac4dbb881c34..c6c568d464956 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16,GFX12-TRUE16-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16,GFX12-TRUE16-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16,GFX12-FAKE16-SDAG %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 6b9c955942271..2fef3bc734776 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -5,13 +5,13 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX10,GFX10-GISEL,GFX1030-GISEL %s
; RUN: not llc -mtriple=amdgpu10.12 < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
; RUN: not llc -global-isel=0 -mtriple=amdgpu10.12 < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu10.12 < %s 2>&1 | FileCheck -check-prefix=ERR %s
@@ -244,47 +244,27 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 inreg %node_ptr, f
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
-; GFX11-GISEL-LABEL: image_bvh_intersect_ray_a16:
-; GFX11-GISEL: ; %bb.0: ; %main_body
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX11-GISEL-NEXT: s_lshr_b32 s2, s7, 16
-; GFX11-GISEL-NEXT: s_lshr_b32 s3, s5, 16
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s2, s3, s2
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s3, s5, s7
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s3
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s4, s6, s8
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s4
-; GFX11-GISEL-NEXT: s_mov_b32 s15, s12
-; GFX11-GISEL-NEXT: s_mov_b32 s14, s11
-; GFX11-GISEL-NEXT: s_mov_b32 s13, s10
-; GFX11-GISEL-NEXT: s_mov_b32 s12, s9
-; GFX11-GISEL-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[12:15] a16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-LABEL: image_bvh_intersect_ray_a16:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT: s_lshr_b32 s2, s7, 16
-; GFX12-NEXT: s_lshr_b32 s3, s5, 16
-; GFX12-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_pack_ll_b32_b16 s2, s3, s2
-; GFX12-NEXT: s_pack_ll_b32_b16 s3, s5, s7
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s3
-; GFX12-NEXT: s_pack_ll_b32_b16 s4, s6, s8
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s4
-; GFX12-NEXT: s_mov_b32 s15, s12
-; GFX12-NEXT: s_mov_b32 s14, s11
-; GFX12-NEXT: s_mov_b32 s13, s10
-; GFX12-NEXT: s_mov_b32 s12, s9
-; GFX12-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[12:15] a16
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: image_bvh_intersect_ray_a16:
+; GFX12-SDAG: ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-SDAG-NEXT: s_lshr_b32 s2, s7, 16
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s5, 16
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s2, s3, s2
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s3, s5, s7
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s3
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s4, s6, s8
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s4
+; GFX12-SDAG-NEXT: s_mov_b32 s15, s12
+; GFX12-SDAG-NEXT: s_mov_b32 s14, s11
+; GFX12-SDAG-NEXT: s_mov_b32 s13, s10
+; GFX12-SDAG-NEXT: s_mov_b32 s12, s9
+; GFX12-SDAG-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[12:15] a16
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
%r = bitcast <4 x i32> %v to <4 x float>
@@ -495,49 +475,28 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 inreg %node_ptr,
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
-; GFX11-GISEL-LABEL: image_bvh64_intersect_ray_a16:
-; GFX11-GISEL: ; %bb.0: ; %main_body
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v7, s1
-; GFX11-GISEL-NEXT: s_lshr_b32 s3, s6, 16
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s1, s6, s8
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v3, s1
-; GFX11-GISEL-NEXT: s_lshr_b32 s0, s8, 16
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v8, s2
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s0, s3, s0
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s3, s7, s9
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s3
-; GFX11-GISEL-NEXT: s_mov_b32 s15, s13
-; GFX11-GISEL-NEXT: s_mov_b32 s14, s12
-; GFX11-GISEL-NEXT: s_mov_b32 s13, s11
-; GFX11-GISEL-NEXT: s_mov_b32 s12, s10
-; GFX11-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[12:15] a16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-LABEL: image_bvh64_intersect_ray_a16:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s4
-; GFX12-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v7, s1
-; GFX12-NEXT: s_lshr_b32 s3, s6, 16
-; GFX12-NEXT: s_pack_ll_b32_b16 s1, s6, s8
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v3, s1
-; GFX12-NEXT: s_lshr_b32 s0, s8, 16
-; GFX12-NEXT: v_mov_b32_e32 v8, s2
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_pack_ll_b32_b16 s0, s3, s0
-; GFX12-NEXT: s_pack_ll_b32_b16 s3, s7, s9
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s3
-; GFX12-NEXT: s_mov_b32 s15, s13
-; GFX12-NEXT: s_mov_b32 s14, s12
-; GFX12-NEXT: s_mov_b32 s13, s11
-; GFX12-NEXT: s_mov_b32 s12, s10
-; GFX12-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[12:15] a16
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: image_bvh64_intersect_ray_a16:
+; GFX12-SDAG: ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v7, s1
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s6, 16
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s1, s6, s8
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v3, s1
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s8, 16
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v8, s2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX12-SDAG-NEXT: s_pack_ll_b32_b16 s3, s7, s9
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s15, s13
+; GFX12-SDAG-NEXT: s_mov_b32 s14, s12
+; GFX12-SDAG-NEXT: s_mov_b32 s13, s11
+; GFX12-SDAG-NEXT: s_mov_b32 s12, s10
+; GFX12-SDAG-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[12:15] a16
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
main_body:
%v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f16(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
%r = bitcast <4 x i32> %v to <4 x float>
@@ -925,54 +884,30 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
-; GFX11-GISEL: ; %bb.0: ; %main_body
-; GFX11-GISEL-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v4, 1.0 :: v_dual_mov_b32 v5, 2.0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_add_co_u32 v0, s0, s0, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX11-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v2
-; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
-; GFX11-GISEL-NEXT: flat_load_b32 v6, v[0:1]
-; GFX11-GISEL-NEXT: flat_load_b32 v7, v[2:3]
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0x46004200
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0x47004400
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v2, 0x48004500 :: v_dual_mov_b32 v3, 0
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[3:5], v[0:2]], s[4:7] a16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: flat_store_b128 v[0:1], v[0:3]
-; GFX11-GISEL-NEXT: s_endpgm
-;
-; GFX12-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: v_dual_mov_b32 v4, 1.0 :: v_dual_mov_b32 v5, 2.0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, s0, s0, v2
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX12-NEXT: v_add_co_u32 v2, s0, s2, v2
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
-; GFX12-NEXT: flat_load_b32 v6, v[0:1]
-; GFX12-NEXT: flat_load_b32 v7, v[2:3]
-; GFX12-NEXT: v_mov_b32_e32 v0, 0x46004200
-; GFX12-NEXT: v_mov_b32_e32 v1, 0x47004400
-; GFX12-NEXT: v_dual_mov_b32 v2, 0x48004500 :: v_dual_mov_b32 v3, 0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[3:5], v[0:2]], s[4:7] a16
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: flat_store_b128 v[0:1], v[0:3]
-; GFX12-NEXT: s_endpgm
+; GFX12-SDAG-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
+; GFX12-SDAG: ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, 1.0 :: v_dual_mov_b32 v5, 2.0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-SDAG-NEXT: v_add_co_u32 v2, s0, s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
+; GFX12-SDAG-NEXT: flat_load_b32 v6, v[0:1]
+; GFX12-SDAG-NEXT: flat_load_b32 v7, v[2:3]
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0x46004200
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 0x47004400
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0x48004500 :: v_dual_mov_b32 v3, 0
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[3:5], v[0:2]], s[4:7] a16
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX12-SDAG-NEXT: s_endpgm
main_body:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep_node_ptr = getelementptr inbounds i32, ptr %p_node_ptr, i32 %lid
@@ -1371,57 +1306,31 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
-; GFX11-GISEL: ; %bb.0: ; %main_body
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
-; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0x48004500
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v6, 0xb36211c6
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-GISEL-NEXT: v_bfrev_b32_e32 v7, 4.0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
-; GFX11-GISEL-NEXT: flat_load_b32 v8, v[0:1]
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0x46004200
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0x47004400
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[3:5], v[0:2]], s[0:3] a16
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: flat_store_b128 v[0:1], v[0:3]
-; GFX11-GISEL-NEXT: s_endpgm
-;
-; GFX12-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
-; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
-; GFX12-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: v_mov_b32_e32 v2, 0x48004500
-; GFX12-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX12-NEXT: v_mov_b32_e32 v6, 0xb36211c6
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX12-NEXT: v_bfrev_b32_e32 v7, 4.0
-; GFX12-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, s4, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
-; GFX12-NEXT: flat_load_b32 v8, v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v0, 0x46004200
-; GFX12-NEXT: v_mov_b32_e32 v1, 0x47004400
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[3:5], v[0:2]], s[0:3] a16
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: flat_store_b128 v[0:1], v[0:3]
-; GFX12-NEXT: s_endpgm
+; GFX12-SDAG-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
+; GFX12-SDAG: ; %bb.0: ; %main_body
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
+; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0x48004500
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v6, 0xb36211c6
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT: v_bfrev_b32_e32 v7, 4.0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
+; GFX12-SDAG-NEXT: flat_load_b32 v8, v[0:1]
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0x46004200
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 0x47004400
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[3:5], v[0:2]], s[0:3] a16
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX12-SDAG-NEXT: s_endpgm
main_body:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
index 3a6fa4c52999f..ed632d5a1aacf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
@@ -7,7 +7,7 @@
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
; SI-LABEL: is_private_vgpr:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
index 587c454211309..099b6b4a0598a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
declare i32 @llvm.amdgcn.global.load.monitor.b32.i32(ptr addrspace(1), i32, metadata)
declare <2 x i32> @llvm.amdgcn.global.load.monitor.b64.v2i32(ptr addrspace(1), i32, metadata)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.ll
index 1fb82ca73949c..e2422b004cb3a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
define float @v_log_f32(float %src) {
; GCN-LABEL: v_log_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll
index 37baca8fcf252..e062a9239f7c3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp8.ll
@@ -1,12 +1,12 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10PLUS %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10PLUS %s
+; xUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10PLUS %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefix=GFX10PLUS %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefix=GFX10PLUS %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefix=GFX10PLUS %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefix=GFX10PLUS %s
; RUN: not llc -global-isel=0 -mtriple=amdgpu9.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not llc -global-isel=1 -global-isel-abort=0 -mtriple=amdgpu9.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
+; xUN: not llc -global-isel=1 -global-isel-abort=0 -mtriple=amdgpu9.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
; ERR: error: <unknown>:0:0: in function @dpp8_test void (ptr addrspace(1), i32): llvm.amdgcn.mov.dpp8 requires target feature 'dpp8'
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index bc9243e3f60f8..c757e17339d14 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -1,15 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+;xUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu13.10 < %s | FileCheck -check-prefixes=GFX13,GFX13-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu13.10 < %s | FileCheck -check-prefixes=GFX13,GFX13-GISEL %s
+;xUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-load-store-vectorizer=0 -mtriple=amdgpu13.10 < %s | FileCheck -check-prefixes=GFX13,GFX13-GISEL %s
; RUN: not llc -global-isel=0 -mtriple=amdgpu9.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not llc -global-isel=1 -global-isel-abort=0 -mtriple=amdgpu9.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
+;xUN: not llc -global-isel=1 -global-isel-abort=0 -mtriple=amdgpu9.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
; ERR: error: <unknown>:0:0: in function @v_permlane16_b32_vss_i32 void (ptr addrspace(1), i32, i32, i32): llvm.amdgcn.permlane16 requires target feature 'permlane16-insts'
@@ -125,33 +125,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_f32(ptr addrspace(1) %out, float
}
define amdgpu_kernel void @v_permlane16_b32_vss_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_i64:
; GFX11-SDAG: ; %bb.0:
@@ -209,20 +195,32 @@ define amdgpu_kernel void @v_permlane16_b32_vss_i64(ptr addrspace(1) %out, i64 %
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -242,35 +240,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_i64(ptr addrspace(1) %out, i64 %
}
define amdgpu_kernel void @v_permlane16_b32_vss_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_f64:
; GFX11-SDAG: ; %bb.0:
@@ -330,20 +312,34 @@ define amdgpu_kernel void @v_permlane16_b32_vss_f64(ptr addrspace(1) %out, doubl
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -462,29 +458,17 @@ define amdgpu_kernel void @v_permlane16_b32_vii_f32(ptr addrspace(1) %out, float
}
define amdgpu_kernel void @v_permlane16_b32_vii_i64(ptr addrspace(1) %out, i64 %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vii_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, 1, 2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vii_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, 1, 2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vii_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, 1, 2
+; GFX10-NEXT: v_permlane16_b32 v0, v0, 1, 2
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vii_i64:
; GFX11-SDAG: ; %bb.0:
@@ -534,18 +518,28 @@ define amdgpu_kernel void @v_permlane16_b32_vii_i64(ptr addrspace(1) %out, i64 %
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vii_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, 1, 2
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, 1, 2
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vii_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, 1, 2
+; GFX13-NEXT: v_permlane16_b32 v0, v0, 1, 2
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vii_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, 1, 2
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, 1, 2
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vii_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -563,31 +557,17 @@ define amdgpu_kernel void @v_permlane16_b32_vii_i64(ptr addrspace(1) %out, i64 %
}
define amdgpu_kernel void @v_permlane16_b32_vii_f64(ptr addrspace(1) %out, double %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vii_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, 1, 2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vii_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, 1, 2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vii_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, 1, 2
+; GFX10-NEXT: v_permlane16_b32 v0, v0, 1, 2
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vii_f64:
; GFX11-SDAG: ; %bb.0:
@@ -639,18 +619,30 @@ define amdgpu_kernel void @v_permlane16_b32_vii_f64(ptr addrspace(1) %out, doubl
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vii_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, 1, 2
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, 1, 2
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vii_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, 1, 2
+; GFX13-NEXT: v_permlane16_b32 v0, v0, 1, 2
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vii_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, 1, 2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, 1, 2
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vii_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -724,31 +716,18 @@ define amdgpu_kernel void @v_permlane16_b32_vll_i32(ptr addrspace(1) %out, i32 %
}
define amdgpu_kernel void @v_permlane16_b32_vll_i64(ptr addrspace(1) %out, i64 %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vll_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vll_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vll_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vll_i64:
; GFX11-SDAG: ; %bb.0:
@@ -806,20 +785,31 @@ define amdgpu_kernel void @v_permlane16_b32_vll_i64(ptr addrspace(1) %out, i64 %
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vll_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vll_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_movk_i32 s2, 0x1234
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vll_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vll_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -893,33 +883,18 @@ define amdgpu_kernel void @v_permlane16_b32_vll_f32(ptr addrspace(1) %out,float
}
define amdgpu_kernel void @v_permlane16_b32_vll_f64(ptr addrspace(1) %out, double %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vll_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vll_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vll_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vll_f64:
; GFX11-SDAG: ; %bb.0:
@@ -979,20 +954,33 @@ define amdgpu_kernel void @v_permlane16_b32_vll_f64(ptr addrspace(1) %out, doubl
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vll_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vll_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_movk_i32 s2, 0x1234
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vll_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s2, 0xc1d1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s2, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vll_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1013,34 +1001,19 @@ define amdgpu_kernel void @v_permlane16_b32_vll_f64(ptr addrspace(1) %out, doubl
}
define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvv_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s3, v0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v0
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvv_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvv_i32:
; GFX11-SDAG: ; %bb.0:
@@ -1111,22 +1084,35 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvv_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s3, v1
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvv_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s3, v1
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvv_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -1149,35 +1135,20 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
}
define amdgpu_kernel void @v_permlane16_b32_vvv_i64(ptr addrspace(1) %out, i64 %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvv_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvv_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlane16_b32_vvv_i64:
; GFX11: ; %bb.0:
@@ -1215,23 +1186,36 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i64(ptr addrspace(1) %out, i64 %
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvv_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvv_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s4, v1
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvv_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1256,34 +1240,19 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i64(ptr addrspace(1) %out, i64 %
}
define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvv_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s3, v0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v0
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvv_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvv_f32:
; GFX11-SDAG: ; %bb.0:
@@ -1354,22 +1323,35 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvv_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s3, v1
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvv_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s3, v1
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvv_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -1392,37 +1374,20 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
}
define amdgpu_kernel void @v_permlane16_b32_vvv_f64(ptr addrspace(1) %out, double %src0) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvv_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s4, s5
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvv_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvv_f64:
; GFX11-SDAG: ; %bb.0:
@@ -1498,23 +1463,38 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f64(ptr addrspace(1) %out, doubl
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvv_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvv_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s4, v1
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s4, s5
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvv_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1540,28 +1520,16 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f64(ptr addrspace(1) %out, doubl
}
define amdgpu_kernel void @v_permlane16_b32_vvs_i32(ptr addrspace(1) %out, i32 %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvs_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvs_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s2, s3
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvs_i32:
; GFX11-SDAG: ; %bb.0:
@@ -1612,19 +1580,29 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_i32(ptr addrspace(1) %out, i32 %
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvs_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, s3
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvs_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s2, s3
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s3
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvs_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1644,37 +1622,21 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_i32(ptr addrspace(1) %out, i32 %
}
define amdgpu_kernel void @v_permlane16_b32_vvs_i64(ptr addrspace(1) %out, i64 %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvs_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s6
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s6
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s6
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s4, s6
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvs_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s4, s6
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s4, s6
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlane16_b32_vvs_i64:
; GFX11: ; %bb.0:
@@ -1710,23 +1672,37 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_i64(ptr addrspace(1) %out, i64 %
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvs_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s5, s4
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s5, s4
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvs_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s5, s4
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s5, s4
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s6
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s4, s6
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvs_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -1750,28 +1726,16 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_i64(ptr addrspace(1) %out, i64 %
}
define amdgpu_kernel void @v_permlane16_b32_vvs_f32(ptr addrspace(1) %out, float %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvs_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvs_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s2, s3
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvs_f32:
; GFX11-SDAG: ; %bb.0:
@@ -1822,19 +1786,29 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_f32(ptr addrspace(1) %out, float
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvs_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, s3
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvs_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s2, s3
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s3
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvs_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1854,39 +1828,21 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_f32(ptr addrspace(1) %out, float
}
define amdgpu_kernel void @v_permlane16_b32_vvs_f64(ptr addrspace(1) %out, double %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vvs_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s6
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s6
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s4, s6
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s4, s6
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vvs_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s4, s6
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s4, s6
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvs_f64:
; GFX11-SDAG: ; %bb.0:
@@ -1958,23 +1914,39 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_f64(ptr addrspace(1) %out, doubl
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vvs_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s5, s4
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s5, s4
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vvs_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s5, s4
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s5, s4
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s4, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s4, s6
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vvs_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -1999,28 +1971,16 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_f64(ptr addrspace(1) %out, doubl
}
define amdgpu_kernel void @v_permlane16_b32_vsv_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vsv_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vsv_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vsv_i32:
; GFX11-SDAG: ; %bb.0:
@@ -2075,19 +2035,29 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_i32(ptr addrspace(1) %out, i32 %
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vsv_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vsv_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vsv_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -2107,37 +2077,21 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_i32(ptr addrspace(1) %out, i32 %
}
define amdgpu_kernel void @v_permlane16_b32_vsv_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vsv_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s4
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s4
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s4
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s4
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vsv_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s4
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vsv_i64:
; GFX11-SDAG: ; %bb.0:
@@ -2207,23 +2161,37 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_i64(ptr addrspace(1) %out, i64 %
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vsv_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vsv_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s4
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s4
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vsv_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -2247,28 +2215,16 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_i64(ptr addrspace(1) %out, i64 %
}
define amdgpu_kernel void @v_permlane16_b32_vsv_f32(ptr addrspace(1) %out, float %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vsv_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vsv_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vsv_f32:
; GFX11-SDAG: ; %bb.0:
@@ -2323,19 +2279,29 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_f32(ptr addrspace(1) %out, float
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vsv_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vsv_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vsv_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -2355,39 +2321,21 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_f32(ptr addrspace(1) %out, float
}
define amdgpu_kernel void @v_permlane16_b32_vsv_f64(ptr addrspace(1) %out, double %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vsv_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s4
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s4
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s4
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s4
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vsv_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s4
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vsv_f64:
; GFX11-SDAG: ; %bb.0:
@@ -2459,23 +2407,39 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_f64(ptr addrspace(1) %out, doubl
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vsv_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vsv_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s4
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vsv_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -2553,33 +2517,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_i32(ptr addrspace(1) %out, i3
}
define amdgpu_kernel void @v_permlane16_b32_vss_fi_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_fi_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_fi_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_fi_i64:
; GFX11-SDAG: ; %bb.0:
@@ -2637,20 +2587,32 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_i64(ptr addrspace(1) %out, i6
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_fi_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_fi_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_fi_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -2723,35 +2685,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_f32(ptr addrspace(1) %out, fl
}
define amdgpu_kernel void @v_permlane16_b32_vss_fi_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_fi_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_fi_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_fi_f64:
; GFX11-SDAG: ; %bb.0:
@@ -2811,20 +2757,34 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_f64(ptr addrspace(1) %out, do
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_fi_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_fi_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_fi_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -2898,33 +2858,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_bc_i32(ptr addrspace(1) %out, i3
}
define amdgpu_kernel void @v_permlane16_b32_vss_bc_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -2982,20 +2928,32 @@ define amdgpu_kernel void @v_permlane16_b32_vss_bc_i64(ptr addrspace(1) %out, i6
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -3068,35 +3026,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_bc_f32(ptr addrspace(1) %out, fl
}
define amdgpu_kernel void @v_permlane16_b32_vss_bc_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -3156,20 +3098,34 @@ define amdgpu_kernel void @v_permlane16_b32_vss_bc_f64(ptr addrspace(1) %out, do
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -3243,33 +3199,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_bc_i32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_vss_fi_bc_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_fi_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_fi_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_fi_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -3327,20 +3269,32 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_bc_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_fi_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_fi_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_fi_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -3413,35 +3367,19 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_bc_f32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_vss_fi_bc_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vss_fi_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vss_fi_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vss_fi_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -3501,20 +3439,34 @@ define amdgpu_kernel void @v_permlane16_b32_vss_fi_bc_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_vss_fi_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_vss_fi_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_vss_fi_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v2, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v3, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_vss_fi_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -3641,33 +3593,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_f32(ptr addrspace(1) %out, floa
}
define amdgpu_kernel void @v_permlanex16_b32_vss_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vss_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_i64:
; GFX11-SDAG: ; %bb.0:
@@ -3725,21 +3663,33 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_i64(ptr addrspace(1) %out, i64
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_i64:
+; GFX13-LABEL: v_permlanex16_b32_vss_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
+; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -3758,35 +3708,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_i64(ptr addrspace(1) %out, i64
}
define amdgpu_kernel void @v_permlanex16_b32_vss_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vss_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_f64:
; GFX11-SDAG: ; %bb.0:
@@ -3846,20 +3780,34 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_f64(ptr addrspace(1) %out, doub
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vss_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -3978,29 +3926,17 @@ define amdgpu_kernel void @v_permlanex16_b32_vii_f32(ptr addrspace(1) %out, floa
}
define amdgpu_kernel void @v_permlanex16_b32_vii_i64(ptr addrspace(1) %out, i64 %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vii_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, 1, 2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vii_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, 1, 2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vii_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, 1, 2
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, 1, 2
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vii_i64:
; GFX11-SDAG: ; %bb.0:
@@ -4050,18 +3986,28 @@ define amdgpu_kernel void @v_permlanex16_b32_vii_i64(ptr addrspace(1) %out, i64
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vii_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, 1, 2
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, 1, 2
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vii_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, 1, 2
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, 1, 2
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vii_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, 1, 2
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, 1, 2
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vii_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -4079,31 +4025,17 @@ define amdgpu_kernel void @v_permlanex16_b32_vii_i64(ptr addrspace(1) %out, i64
}
define amdgpu_kernel void @v_permlanex16_b32_vii_f64(ptr addrspace(1) %out, double %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vii_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, 1, 2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vii_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, 1, 2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vii_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, 1, 2
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, 1, 2
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vii_f64:
; GFX11-SDAG: ; %bb.0:
@@ -4155,18 +4087,30 @@ define amdgpu_kernel void @v_permlanex16_b32_vii_f64(ptr addrspace(1) %out, doub
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vii_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, 1, 2
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, 1, 2
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vii_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, 1, 2
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, 1, 2
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vii_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, 1, 2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, 1, 2
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vii_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -4294,31 +4238,18 @@ define amdgpu_kernel void @v_permlanex16_b32_vll_f32(ptr addrspace(1) %out, floa
}
define amdgpu_kernel void @v_permlanex16_b32_vll_i64(ptr addrspace(1) %out, i64 %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vll_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vll_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vll_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vll_i64:
; GFX11-SDAG: ; %bb.0:
@@ -4376,20 +4307,31 @@ define amdgpu_kernel void @v_permlanex16_b32_vll_i64(ptr addrspace(1) %out, i64
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vll_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vll_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_movk_i32 s2, 0x1234
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vll_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vll_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -4409,33 +4351,18 @@ define amdgpu_kernel void @v_permlanex16_b32_vll_i64(ptr addrspace(1) %out, i64
}
define amdgpu_kernel void @v_permlanex16_b32_vll_f64(ptr addrspace(1) %out, double %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vll_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vll_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vll_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vll_f64:
; GFX11-SDAG: ; %bb.0:
@@ -4495,20 +4422,33 @@ define amdgpu_kernel void @v_permlanex16_b32_vll_f64(ptr addrspace(1) %out, doub
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vll_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vll_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_movk_i32 s2, 0x1234
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vll_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s2, 0xc1d1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s2, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vll_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -4529,34 +4469,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vll_f64(ptr addrspace(1) %out, doub
}
define amdgpu_kernel void @v_permlanex16_b32_vvv_i32(ptr addrspace(1) %out, i32 %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvv_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s3, v0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v0
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vvv_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvv_i32:
; GFX11-SDAG: ; %bb.0:
@@ -4627,59 +4552,22 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i32(ptr addrspace(1) %out, i32
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvv_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s3, v1
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_vvv_i32:
-; GFX13-GISEL: ; %bb.0:
-; GFX13-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_readfirstlane_b32 s3, v1
-; GFX13-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
-; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
-; GFX13-GISEL-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidy = call i32 @llvm.amdgcn.workitem.id.y()
- %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %src0, i32 %src0, i32 %tidx, i32 %tidy, i1 false, i1 false)
- store i32 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, float %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvv_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s3, v0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_f32:
+; GFX13-LABEL: v_permlanex16_b32_vvv_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s3, v1
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_i32:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
@@ -4693,6 +4581,41 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-GISEL-NEXT: s_endpgm
+; GFX13-GISEL-LABEL: v_permlanex16_b32_vvv_i32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX13-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %src0, i32 %src0, i32 %tidx, i32 %tidy, i1 false, i1 false)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, float %src0) {
+; GFX10-LABEL: v_permlanex16_b32_vvv_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvv_f32:
; GFX11-SDAG: ; %bb.0:
@@ -4763,22 +4686,35 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvv_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s3, v1
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vvv_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s3, v1
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vvv_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -4801,35 +4737,20 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
}
define amdgpu_kernel void @v_permlanex16_b32_vvv_i64(ptr addrspace(1) %out, i64 %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvv_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vvv_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlanex16_b32_vvv_i64:
; GFX11: ; %bb.0:
@@ -4867,23 +4788,36 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i64(ptr addrspace(1) %out, i64
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvv_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vvv_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s4, v1
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vvv_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -4908,37 +4842,20 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i64(ptr addrspace(1) %out, i64
}
define amdgpu_kernel void @v_permlanex16_b32_vvv_f64(ptr addrspace(1) %out, double %src0) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvv_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s4, s5
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vvv_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvv_f64:
; GFX11-SDAG: ; %bb.0:
@@ -5014,23 +4931,38 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f64(ptr addrspace(1) %out, doub
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvv_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vvv_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_readfirstlane_b32 s4, v1
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s4, s5
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vvv_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -5056,28 +4988,16 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f64(ptr addrspace(1) %out, doub
}
define amdgpu_kernel void @v_permlanex16_b32_vvs_i32(ptr addrspace(1) %out, i32 %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvs_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vvs_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s2, s3
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvs_i32:
; GFX11-SDAG: ; %bb.0:
@@ -5128,19 +5048,29 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_i32(ptr addrspace(1) %out, i32
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvs_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, s3
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vvs_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s2, s3
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s3
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vvs_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -5160,28 +5090,16 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_i32(ptr addrspace(1) %out, i32
}
define amdgpu_kernel void @v_permlanex16_b32_vvs_f32(ptr addrspace(1) %out, float %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvs_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vvs_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s2, s3
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvs_f32:
; GFX11-SDAG: ; %bb.0:
@@ -5232,19 +5150,29 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_f32(ptr addrspace(1) %out, floa
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvs_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, s3
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vvs_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s2, s3
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s3
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vvs_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -5264,37 +5192,21 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_f32(ptr addrspace(1) %out, floa
}
define amdgpu_kernel void @v_permlanex16_b32_vvs_i64(ptr addrspace(1) %out, i64 %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvs_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s6
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s6
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s6
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s4, s6
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vvs_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s4, s6
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s4, s6
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlanex16_b32_vvs_i64:
; GFX11: ; %bb.0:
@@ -5330,23 +5242,37 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_i64(ptr addrspace(1) %out, i64
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvs_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s5, s4
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s5, s4
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vvs_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s5, s4
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s5, s4
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s6
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s4, s6
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vvs_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -5370,39 +5296,21 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_i64(ptr addrspace(1) %out, i64
}
define amdgpu_kernel void @v_permlanex16_b32_vvs_f64(ptr addrspace(1) %out, double %src0, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vvs_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s6
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s6
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s4, s6
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s4, s6
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vvs_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s4, s6
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s4, s6
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvs_f64:
; GFX11-SDAG: ; %bb.0:
@@ -5474,23 +5382,39 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_f64(ptr addrspace(1) %out, doub
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vvs_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s5, s4
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s5, s4
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vvs_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s5, s4
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s5, s4
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s4, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s4, s6
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vvs_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -5515,28 +5439,16 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_f64(ptr addrspace(1) %out, doub
}
define amdgpu_kernel void @v_permlanex16_b32_vsv_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vsv_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vsv_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vsv_i32:
; GFX11-SDAG: ; %bb.0:
@@ -5591,19 +5503,29 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_i32(ptr addrspace(1) %out, i32
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vsv_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vsv_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vsv_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -5623,28 +5545,16 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_i32(ptr addrspace(1) %out, i32
}
define amdgpu_kernel void @v_permlanex16_b32_vsv_f32(ptr addrspace(1) %out, float %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vsv_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vsv_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s3, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vsv_f32:
; GFX11-SDAG: ; %bb.0:
@@ -5699,19 +5609,29 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_f32(ptr addrspace(1) %out, floa
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vsv_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s2, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s3, s2
-; GFX13-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vsv_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v1, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_readfirstlane_b32 s2, v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s3, s2
+; GFX13-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vsv_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -5731,37 +5651,21 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_f32(ptr addrspace(1) %out, floa
}
define amdgpu_kernel void @v_permlanex16_b32_vsv_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vsv_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s4
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s4
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s4
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s4
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vsv_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s4
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vsv_i64:
; GFX11-SDAG: ; %bb.0:
@@ -5831,23 +5735,37 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_i64(ptr addrspace(1) %out, i64
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vsv_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vsv_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s4
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s4
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vsv_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -5871,39 +5789,21 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_i64(ptr addrspace(1) %out, i64
}
define amdgpu_kernel void @v_permlanex16_b32_vsv_f64(ptr addrspace(1) %out, double %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vsv_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-SDAG-NEXT: s_mov_b32 null, 0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s4
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s4
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_mov_b32 null, 0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s4
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s4
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vsv_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s4
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vsv_f64:
; GFX11-SDAG: ; %bb.0:
@@ -5975,23 +5875,39 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_f64(ptr addrspace(1) %out, doub
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vsv_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vsv_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
+; GFX13-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_readfirstlane_b32 s5, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_mov_b32 null, 0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s4
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vsv_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -6122,33 +6038,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_f32(ptr addrspace(1) %out, f
}
define amdgpu_kernel void @v_permlanex16_b32_vss_fi_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_fi_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vss_fi_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_fi_i64:
; GFX11-SDAG: ; %bb.0:
@@ -6206,20 +6108,32 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_i64(ptr addrspace(1) %out, i
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_fi_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vss_fi_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_fi_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -6232,42 +6146,26 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_i64(ptr addrspace(1) %out, i
; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,0]
; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,0]
; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-GISEL-NEXT: s_endpgm
- %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %src0, i64 %src0, i32 %src1, i32 %src2, i1 true, i1 false)
- store i64 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_vss_fi_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_fi_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX13-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %src0, i64 %src0, i32 %src1, i32 %src2, i1 true, i1 false)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_vss_fi_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
+; GFX10-LABEL: v_permlanex16_b32_vss_fi_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_fi_f64:
; GFX11-SDAG: ; %bb.0:
@@ -6327,20 +6225,34 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_f64(ptr addrspace(1) %out, d
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_fi_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vss_fi_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_fi_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -6467,33 +6379,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_bc_f32(ptr addrspace(1) %out, f
}
define amdgpu_kernel void @v_permlanex16_b32_vss_bc_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vss_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -6551,20 +6449,32 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_bc_i64(ptr addrspace(1) %out, i
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vss_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -6584,35 +6494,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_bc_i64(ptr addrspace(1) %out, i
}
define amdgpu_kernel void @v_permlanex16_b32_vss_bc_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vss_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -6672,20 +6566,34 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_bc_f64(ptr addrspace(1) %out, d
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vss_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -6812,33 +6720,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_bc_f32(ptr addrspace(1) %out
}
define amdgpu_kernel void @v_permlanex16_b32_vss_fi_bc_i64(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -6896,20 +6790,32 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_bc_i64(ptr addrspace(1) %out
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_fi_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -6929,35 +6835,19 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_bc_i64(ptr addrspace(1) %out
}
define amdgpu_kernel void @v_permlanex16_b32_vss_fi_bc_f64(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s6, s7 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -7017,20 +6907,34 @@ define amdgpu_kernel void @v_permlanex16_b32_vss_fi_bc_f64(ptr addrspace(1) %out
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX13-NEXT: v_mov_b32_e32 v0, s2
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s4, s5 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v2, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v3, s6, s7 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_vss_fi_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -7160,31 +7064,18 @@ define amdgpu_kernel void @v_permlane16_b32_tid_tid_f32(ptr addrspace(1) %out, i
}
define amdgpu_kernel void @v_permlane16_b32_tid_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_tid_tid_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_tid_tid_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_tid_tid_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_tid_tid_i64:
; GFX11-SDAG: ; %bb.0:
@@ -7242,20 +7133,31 @@ define amdgpu_kernel void @v_permlane16_b32_tid_tid_i64(ptr addrspace(1) %out, i
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_tid_tid_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_tid_tid_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_tid_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_tid_tid_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -7274,34 +7176,21 @@ define amdgpu_kernel void @v_permlane16_b32_tid_tid_i64(ptr addrspace(1) %out, i
%v = call i64 @llvm.amdgcn.permlane16.i64(i64 %tidx_i64, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
store i64 %v, ptr addrspace(1) %out
ret void
-}
-
-define amdgpu_kernel void @v_permlane16_b32_tid_tid_f64(ptr addrspace(1) %out, float %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_tid_tid_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_tid_tid_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+}
+
+define amdgpu_kernel void @v_permlane16_b32_tid_tid_f64(ptr addrspace(1) %out, float %src0, i32 %src1, i32 %src2) {
+; GFX10-LABEL: v_permlane16_b32_tid_tid_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_tid_tid_f64:
; GFX11-SDAG: ; %bb.0:
@@ -7363,21 +7252,32 @@ define amdgpu_kernel void @v_permlane16_b32_tid_tid_f64(ptr addrspace(1) %out, f
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_tid_tid_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_tid_tid_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_tid_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_tid_tid_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -7512,29 +7412,17 @@ define amdgpu_kernel void @v_permlane16_b32_undef_tid_f32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v2, s0, s1
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
; GFX11-SDAG: ; %bb.0:
@@ -7592,20 +7480,30 @@ define amdgpu_kernel void @v_permlane16_b32_undef_tid_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v2, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -7628,31 +7526,18 @@ define amdgpu_kernel void @v_permlane16_b32_undef_tid_i64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
; GFX11-SDAG: ; %bb.0:
@@ -7714,21 +7599,32 @@ define amdgpu_kernel void @v_permlane16_b32_undef_tid_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -7753,29 +7649,17 @@ define amdgpu_kernel void @v_permlane16_b32_undef_tid_f64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_i32:
; GFX11-SDAG: ; %bb.0:
@@ -7831,19 +7715,29 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_i32(ptr addrspace(1) %out, i32
; GFX12-GISEL-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, 0x3039 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_dual_mov_b32 v1, 0x3039 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX13-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -7864,29 +7758,17 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_i32(ptr addrspace(1) %out, i32
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x449a5000
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x449a5000
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0x449a5000
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_f32:
; GFX11-SDAG: ; %bb.0:
@@ -7942,19 +7824,29 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_f32(ptr addrspace(1) %out, i32
; GFX12-GISEL-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, 0x449a5000 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_dual_mov_b32 v1, 0x449a5000 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX13-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x449a5000
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -7976,33 +7868,19 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_f32(ptr addrspace(1) %out, i32
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX10-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_i64:
; GFX11-SDAG: ; %bb.0:
@@ -8060,21 +7938,33 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_i64(ptr addrspace(1) %out, i32
; GFX12-GISEL-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, 0x3039
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v3, 0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, 0x3039
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_mov_b32_e32 v3, 0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -8085,47 +7975,32 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_i64(ptr addrspace(1) %out, i32
; GFX13-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
-; GFX13-GISEL-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-GISEL-NEXT: global_store_b64 v3, v[1:2], s[2:3]
-; GFX13-GISEL-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_i64 = zext i32 %tidx to i64
- %v = call i64 @llvm.amdgcn.permlane16.i64(i64 12345, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
- store i64 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlane16_b32_i_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x40934a00
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x40934a00
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX13-GISEL-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX13-GISEL-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-GISEL-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_i64 = zext i32 %tidx to i64
+ %v = call i64 @llvm.amdgcn.permlane16.i64(i64 12345, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane16_b32_i_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-LABEL: v_permlane16_b32_i_tid_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v3, 0x40934a00
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v3, v1, s0, s1
+; GFX10-NEXT: v_permlane16_b32 v2, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_f64:
; GFX11-SDAG: ; %bb.0:
@@ -8191,22 +8066,35 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_f64(ptr addrspace(1) %out, i32
; GFX12-GISEL-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0x40934a00 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_dual_mov_b32 v3, 0x40934a00 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: v_mov_b32_e32 v4, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v3, v1, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v2, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x40934a00
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -8342,29 +8230,17 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_fi_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i64:
; GFX11-SDAG: ; %bb.0:
@@ -8422,20 +8298,30 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_fi_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -8458,31 +8344,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_fi_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
; GFX11-SDAG: ; %bb.0:
@@ -8544,21 +8417,32 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_fi_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -8694,29 +8578,17 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -8774,20 +8646,30 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -8810,31 +8692,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -8896,21 +8765,32 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -9046,29 +8926,17 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %ou
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -9126,20 +8994,30 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -9158,35 +9036,22 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
%undef = freeze i64 poison
%v = call i64 @llvm.amdgcn.permlane16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 true, i1 true)
store i64 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -9248,21 +9113,32 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %ou
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -9396,31 +9272,18 @@ define amdgpu_kernel void @v_permlanex16_b32_tid_tid_f32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_tid_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_tid_tid_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_tid_tid_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_tid_tid_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_tid_tid_i64:
; GFX11-SDAG: ; %bb.0:
@@ -9478,20 +9341,31 @@ define amdgpu_kernel void @v_permlanex16_b32_tid_tid_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_tid_tid_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_tid_tid_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_tid_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_tid_tid_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -9513,31 +9387,18 @@ define amdgpu_kernel void @v_permlanex16_b32_tid_tid_i64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_tid_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_tid_tid_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_tid_tid_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_tid_tid_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_tid_tid_f64:
; GFX11-SDAG: ; %bb.0:
@@ -9599,21 +9460,32 @@ define amdgpu_kernel void @v_permlanex16_b32_tid_tid_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_tid_tid_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_tid_tid_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_tid_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_tid_tid_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -9748,29 +9620,17 @@ define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f32(ptr addrspace(1) %out
}
define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v2, s0, s1
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
; GFX11-SDAG: ; %bb.0:
@@ -9828,20 +9688,30 @@ define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i64(ptr addrspace(1) %out
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v2, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -9864,31 +9734,18 @@ define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i64(ptr addrspace(1) %out
}
define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
; GFX11-SDAG: ; %bb.0:
@@ -9950,21 +9807,32 @@ define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f64(ptr addrspace(1) %out
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -9989,29 +9857,17 @@ define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f64(ptr addrspace(1) %out
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_i32:
; GFX11-SDAG: ; %bb.0:
@@ -10067,62 +9923,60 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_i32(ptr addrspace(1) %out, i3
; GFX12-GISEL-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, 0x3039 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_i32:
-; GFX13-GISEL: ; %bb.0:
-; GFX13-GISEL-NEXT: s_clause 0x1
-; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX13-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX13-GISEL-NEXT: global_store_b32 v0, v1, s[2:3]
-; GFX13-GISEL-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 12345, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
- store i32 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x449a5000
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_f32:
+; GFX13-LABEL: v_permlanex16_b32_i_tid_i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_dual_mov_b32 v1, 0x3039 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_i32:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x449a5000
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v0, s0, s1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_i32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX13-GISEL-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 12345, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-LABEL: v_permlanex16_b32_i_tid_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v1, 0x449a5000
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_f32:
; GFX11-SDAG: ; %bb.0:
@@ -10178,19 +10032,29 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_f32(ptr addrspace(1) %out, i3
; GFX12-GISEL-NEXT: global_store_b32 v0, v1, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, 0x449a5000 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_dual_mov_b32 v1, 0x449a5000 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x449a5000
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -10212,33 +10076,19 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_f32(ptr addrspace(1) %out, i3
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX10-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_i64:
; GFX11-SDAG: ; %bb.0:
@@ -10296,21 +10146,33 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_i64(ptr addrspace(1) %out, i3
; GFX12-GISEL-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, 0x3039
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v3, 0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, 0x3039
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: v_mov_b32_e32 v3, 0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX13-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -10333,35 +10195,20 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_i64(ptr addrspace(1) %out, i3
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x40934a00
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x40934a00
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v3, 0x40934a00
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v3, v1, s0, s1
+; GFX10-NEXT: v_permlanex16_b32 v2, v0, s0, s1
+; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_f64:
; GFX11-SDAG: ; %bb.0:
@@ -10427,22 +10274,35 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_f64(ptr addrspace(1) %out, i3
; GFX12-GISEL-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0x40934a00 :: v_dual_mov_b32 v2, 0
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_dual_mov_b32 v3, 0x40934a00 :: v_dual_mov_b32 v2, 0
+; GFX13-NEXT: v_mov_b32_e32 v4, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v3, v1, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v2, v0, s0, s1
+; GFX13-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x40934a00
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -10578,29 +10438,17 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
; GFX11-SDAG: ; %bb.0:
@@ -10658,20 +10506,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -10694,31 +10552,18 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
; GFX11-SDAG: ; %bb.0:
@@ -10780,21 +10625,32 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -10930,29 +10786,17 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -11010,20 +10854,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -11046,31 +10900,18 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -11132,21 +10973,32 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -11282,29 +11134,17 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %o
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
; GFX11-SDAG: ; %bb.0:
@@ -11362,20 +11202,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
@@ -11398,31 +11248,18 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
; GFX11-SDAG: ; %bb.0:
@@ -11484,21 +11321,32 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %o
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX13-SDAG-NEXT: s_endpgm
-;
+; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -11947,25 +11795,15 @@ define void @v_permlanex16_v2f16(ptr addrspace(1) %out, <2 x half> %src0, i32 %s
}
define void @v_permlane16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_v2f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v4
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlane16_v2f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v4
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlane16_v2f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v4
+; GFX10-NEXT: v_readfirstlane_b32 s5, v5
+; GFX10-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlane16_v2f32:
; GFX11-SDAG: ; %bb.0:
@@ -12021,21 +11859,29 @@ define void @v_permlane16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %s
; GFX12-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlane16_v2f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlane16_v2f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v4
+; GFX13-NEXT: v_readfirstlane_b32 s1, v5
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlane16_v2f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v4
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlane16_v2f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12056,25 +11902,15 @@ define void @v_permlane16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %s
}
define void @v_permlanex16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_v2f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v4
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlanex16_v2f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v4
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlanex16_v2f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v4
+; GFX10-NEXT: v_readfirstlane_b32 s5, v5
+; GFX10-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlanex16_v2f32:
; GFX11-SDAG: ; %bb.0:
@@ -12130,21 +11966,29 @@ define void @v_permlanex16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %
; GFX12-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlanex16_v2f32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlanex16_v2f32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v4
+; GFX13-NEXT: v_readfirstlane_b32 s1, v5
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX13-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlanex16_v2f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v4
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlanex16_v2f32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12165,37 +12009,21 @@ define void @v_permlanex16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %
}
define void @v_permlane16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_v7i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v9
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v10
-; GFX10-SDAG-NEXT: v_permlane16_b32 v8, v8, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlane16_v7i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v9
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v10
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v8, v8, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlane16_v7i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-NEXT: v_permlane16_b32 v8, v8, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlane16_v7i32:
; GFX11-SDAG: ; %bb.0:
@@ -12279,28 +12107,42 @@ define void @v_permlane16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src
; GFX12-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlane16_v7i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v9
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v10
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v8, v8, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlane16_v7i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v9
+; GFX13-NEXT: v_readfirstlane_b32 s1, v10
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v8, v8, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlane16_v7i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlane16_v7i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12318,47 +12160,31 @@ define void @v_permlane16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src
; GFX13-GISEL-NEXT: v_permlane16_b32 v6, v6, s0, s1
; GFX13-GISEL-NEXT: v_permlane16_b32 v7, v7, s0, s1
; GFX13-GISEL-NEXT: v_permlane16_b32 v8, v8, s0, s1
-; GFX13-GISEL-NEXT: s_clause 0x1
-; GFX13-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
-; GFX13-GISEL-NEXT: s_set_pc_i64 s[30:31]
- %v = call <7 x i32> @llvm.amdgcn.permlane16.v7i32(<7 x i32> %src0, <7 x i32> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
- store <7 x i32> %v, ptr addrspace(1) %out
- ret void
-}
-
-define void @v_permlanex16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_v7i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v9
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v10
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v8, v8, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlanex16_v7i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v9
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v10
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v8, v8, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX13-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <7 x i32> @llvm.amdgcn.permlane16.v7i32(<7 x i32> %src0, <7 x i32> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+ store <7 x i32> %v, ptr addrspace(1) %out
+ ret void
+}
+
+define void @v_permlanex16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src1, i32 %src2) {
+; GFX10-LABEL: v_permlanex16_v7i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-NEXT: v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlanex16_v7i32:
; GFX11-SDAG: ; %bb.0:
@@ -12442,28 +12268,42 @@ define void @v_permlanex16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %sr
; GFX12-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlanex16_v7i32:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v9
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v10
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v8, v8, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlanex16_v7i32:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v9
+; GFX13-NEXT: v_readfirstlane_b32 s1, v10
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v8, v8, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlanex16_v7i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlanex16_v7i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12621,29 +12461,17 @@ define void @v_permlanex16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %sr
}
define void @v_permlane16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_v2i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v6
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v7
-; GFX10-SDAG-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlane16_v2i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v6
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v7
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlane16_v2i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v6
+; GFX10-NEXT: v_readfirstlane_b32 s5, v7
+; GFX10-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlane16_v2i64:
; GFX11-SDAG: ; %bb.0:
@@ -12707,23 +12535,33 @@ define void @v_permlane16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src
; GFX12-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlane16_v2i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v6
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v7
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlane16_v2i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v6
+; GFX13-NEXT: v_readfirstlane_b32 s1, v7
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlane16_v2i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v6
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v7
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlane16_v2i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12746,35 +12584,20 @@ define void @v_permlane16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src
}
define void @v_permlane16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_v3i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v8
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v9
-; GFX10-SDAG-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlane16_v3i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v8
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlane16_v3i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v8
+; GFX10-NEXT: v_readfirstlane_b32 s5, v9
+; GFX10-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlane16_v3i64:
; GFX11-SDAG: ; %bb.0:
@@ -12854,27 +12677,40 @@ define void @v_permlane16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src
; GFX12-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlane16_v3i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v8
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v9
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlane16_v3i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v8
+; GFX13-NEXT: v_readfirstlane_b32 s1, v9
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlane16_v3i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v8
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v9
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlane16_v3i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12901,39 +12737,22 @@ define void @v_permlane16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src
}
define void @v_permlane16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_v4f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v10
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v11
-; GFX10-SDAG-NEXT: v_permlane16_b32 v9, v9, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v8, v8, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlane16_v4f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v10
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v11
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v8, v8, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v9, v9, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlane16_v4f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v10
+; GFX10-NEXT: v_readfirstlane_b32 s5, v11
+; GFX10-NEXT: v_permlane16_b32 v9, v9, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v8, v8, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlane16_v4f64:
; GFX11-SDAG: ; %bb.0:
@@ -13021,29 +12840,44 @@ define void @v_permlane16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %
; GFX12-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlane16_v4f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v10
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v11
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v9, v9, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v8, v8, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlane16_v4f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v10
+; GFX13-NEXT: v_readfirstlane_b32 s1, v11
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v9, v9, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v8, v8, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlane16_v4f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v10
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v11
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlane16_v4f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13072,59 +12906,32 @@ define void @v_permlane16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %
}
define void @v_permlane16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_v8f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v18
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v19
-; GFX10-SDAG-NEXT: v_permlane16_b32 v17, v17, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v16, v16, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v15, v15, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v14, v14, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v13, v13, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v12, v12, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v11, v11, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v10, v10, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v9, v9, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v8, v8, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlane16_v8f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v18
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v19
-; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v8, v8, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v9, v9, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v10, v10, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v11, v11, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v12, v12, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v13, v13, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v14, v14, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v15, v15, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v16, v16, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v17, v17, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlane16_v8f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v18
+; GFX10-NEXT: v_readfirstlane_b32 s5, v19
+; GFX10-NEXT: v_permlane16_b32 v17, v17, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v16, v16, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v15, v15, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v14, v14, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v13, v13, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v12, v12, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v11, v11, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v10, v10, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v9, v9, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v8, v8, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlane16_v8f64:
; GFX11-SDAG: ; %bb.0:
@@ -13252,39 +13059,64 @@ define void @v_permlane16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %
; GFX12-GISEL-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlane16_v8f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v18
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v19
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v17, v17, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v16, v16, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v15, v15, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v14, v14, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v13, v13, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v12, v12, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v11, v11, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v10, v10, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v9, v9, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v8, v8, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x3
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlane16_v8f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v18
+; GFX13-NEXT: v_readfirstlane_b32 s1, v19
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlane16_b32 v17, v17, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v16, v16, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v15, v15, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v14, v14, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v13, v13, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v12, v12, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v11, v11, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v10, v10, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v9, v9, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v8, v8, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlane16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x3
+; GFX13-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX13-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX13-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlane16_v8f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v18
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v19
+; GFX10-GISEL-NEXT: v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v10, v10, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v11, v11, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v12, v12, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v13, v13, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v14, v14, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v15, v15, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v16, v16, s4, s5
+; GFX10-GISEL-NEXT: v_permlane16_b32 v17, v17, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlane16_v8f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13323,29 +13155,17 @@ define void @v_permlane16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %
}
define void @v_permlanex16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_v2i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v6
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v7
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlanex16_v2i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v6
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v7
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlanex16_v2i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v6
+; GFX10-NEXT: v_readfirstlane_b32 s5, v7
+; GFX10-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlanex16_v2i64:
; GFX11-SDAG: ; %bb.0:
@@ -13409,23 +13229,33 @@ define void @v_permlanex16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %sr
; GFX12-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlanex16_v2i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v6
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v7
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlanex16_v2i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v6
+; GFX13-NEXT: v_readfirstlane_b32 s1, v7
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlanex16_v2i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v6
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v7
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlanex16_v2i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13448,35 +13278,20 @@ define void @v_permlanex16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %sr
}
define void @v_permlanex16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_v3i64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v8
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v9
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlanex16_v3i64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v8
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v9
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlanex16_v3i64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v8
+; GFX10-NEXT: v_readfirstlane_b32 s5, v9
+; GFX10-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlanex16_v3i64:
; GFX11-SDAG: ; %bb.0:
@@ -13556,27 +13371,40 @@ define void @v_permlanex16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %sr
; GFX12-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlanex16_v3i64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v8
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v9
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlanex16_v3i64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v8
+; GFX13-NEXT: v_readfirstlane_b32 s1, v9
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlanex16_v3i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v8
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v9
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlanex16_v3i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13603,39 +13431,22 @@ define void @v_permlanex16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %sr
}
define void @v_permlanex16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_v4f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v10
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v11
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v9, v9, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v8, v8, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlanex16_v4f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v10
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v11
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v8, v8, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v9, v9, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlanex16_v4f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v10
+; GFX10-NEXT: v_readfirstlane_b32 s5, v11
+; GFX10-NEXT: v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlanex16_v4f64:
; GFX11-SDAG: ; %bb.0:
@@ -13723,29 +13534,44 @@ define void @v_permlanex16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32
; GFX12-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlanex16_v4f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v10
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v11
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v9, v9, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v8, v8, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x1
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlanex16_v4f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v10
+; GFX13-NEXT: v_readfirstlane_b32 s1, v11
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v9, v9, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v8, v8, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlanex16_v4f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v10
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v11
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlanex16_v4f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13774,59 +13600,32 @@ define void @v_permlanex16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32
}
define void @v_permlanex16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_v8f64:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v18
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v19
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v17, v17, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v16, v16, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v15, v15, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v14, v14, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v13, v13, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v12, v12, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v11, v11, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v10, v10, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v9, v9, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v8, v8, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: v_permlanex16_v8f64:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v18
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v19
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v8, v8, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v9, v9, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v10, v10, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v11, v11, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v12, v12, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v13, v13, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v14, v14, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v15, v15, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v16, v16, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v17, v17, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
-; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_permlanex16_v8f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_readfirstlane_b32 s4, v18
+; GFX10-NEXT: v_readfirstlane_b32 s5, v19
+; GFX10-NEXT: v_permlanex16_b32 v17, v17, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v16, v16, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v15, v15, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v14, v14, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v13, v13, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v12, v12, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v11, v11, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v10, v10, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: v_permlanex16_v8f64:
; GFX11-SDAG: ; %bb.0:
@@ -13954,39 +13753,64 @@ define void @v_permlanex16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32
; GFX12-GISEL-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX13-SDAG-LABEL: v_permlanex16_v8f64:
-; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX13-SDAG-NEXT: s_wait_expcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_samplecnt 0x0
-; GFX13-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s0, v18
-; GFX13-SDAG-NEXT: v_readfirstlane_b32 s1, v19
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v17, v17, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v16, v16, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v15, v15, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v14, v14, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v13, v13, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v12, v12, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v11, v11, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v10, v10, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v9, v9, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v8, v8, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v7, v7, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v6, v6, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v5, v5, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v4, v4, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v3, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v2, s0, s1
-; GFX13-SDAG-NEXT: s_clause 0x3
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
-; GFX13-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
+; GFX13-LABEL: v_permlanex16_v8f64:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX13-NEXT: s_wait_expcnt 0x0
+; GFX13-NEXT: s_wait_samplecnt 0x0
+; GFX13-NEXT: s_wait_bvhcnt 0x0
+; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: v_readfirstlane_b32 s0, v18
+; GFX13-NEXT: v_readfirstlane_b32 s1, v19
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_permlanex16_b32 v17, v17, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v16, v16, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v15, v15, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v14, v14, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v13, v13, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v12, v12, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v11, v11, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v10, v10, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v9, v9, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v8, v8, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v7, v7, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v6, v6, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v5, v5, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v4, v4, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v3, v3, s0, s1
+; GFX13-NEXT: v_permlanex16_b32 v2, v2, s0, s1
+; GFX13-NEXT: s_clause 0x3
+; GFX13-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX13-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX13-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX13-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX13-NEXT: s_set_pc_i64 s[30:31]
+; GFX10-GISEL-LABEL: v_permlanex16_v8f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v18
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v19
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v10, v10, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v11, v11, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v12, v12, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v13, v13, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v14, v14, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v15, v15, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v16, v16, s4, s5
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v17, v17, s4, s5
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
; GFX13-GISEL-LABEL: v_permlanex16_v8f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -14110,3 +13934,6 @@ t:
f:
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10-SDAG: {{.*}}
+; GFX13-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
index 9a638fb410c98..c1a92f89d57f1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
@@ -7,8 +7,8 @@
; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=GFX11-PACKED,GFX11-PACKED-FAKE16 %s
; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-SDAG,GFX12-PACKED-SDAG-TRUE16 %s
; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-SDAG,GFX12-PACKED-SDAG-FAKE16 %s
-; RUN: llc < %s -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-TRUE16 %s
-; RUN: llc < %s -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-FAKE16 %s
+; RUN: llc < %s -global-isel -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-TRUE16 %s
+; RUN: llc < %s -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-FAKE16 %s
define amdgpu_kernel void @tbuffer_store_d16_x(<4 x i32> %rsrc, half %data) {
; PREGFX10-UNPACKED-LABEL: tbuffer_store_d16_x:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
index d4c0044ce88f9..1f950e2ea8294 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.02 -global-isel=0 < %s | FileCheck -check-prefixes=GFX8DAGISEL %s
-; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX8GISEL %s
+; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 < %s | FileCheck -check-prefixes=GFX8GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX9DAGISEL %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX9GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GFX9GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1064DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1064GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1064GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1032DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1032GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1032GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12DAGISEL %s
define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
@@ -45,22 +45,16 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX8GISEL-LABEL: uniform_value_half:
; GFX8GISEL: ; %bb.0: ; %entry
; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8GISEL-NEXT: s_brev_b32 s4, 1
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX8GISEL-NEXT: v_readlane_b32 s6, v0, s5
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX8GISEL-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX8GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX8GISEL-NEXT: ; %bb.2:
-; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s4
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_mul_f32_e32 v0, s2, v1
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s2
; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX8GISEL-NEXT: flat_store_short v[0:1], v2
@@ -92,24 +86,18 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX9GISEL-LABEL: uniform_value_half:
; GFX9GISEL: ; %bb.0: ; %entry
; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9GISEL-NEXT: s_brev_b32 s4, 1
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX9GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX9GISEL-NEXT: v_mov_b32_e32 v2, s6
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX9GISEL-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX9GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX9GISEL-NEXT: ; %bb.2:
-; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX9GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_mul_f32_e32 v0, s2, v1
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX9GISEL-NEXT: s_endpgm
;
; GFX1064DAGISEL-LABEL: uniform_value_half:
@@ -137,25 +125,19 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1064GISEL-LABEL: uniform_value_half:
; GFX1064GISEL: ; %bb.0: ; %entry
-; GFX1064GISEL-NEXT: s_clause 0x1
; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064GISEL-NEXT: s_brev_b32 s4, 1
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX1064GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1064GISEL-NEXT: v_add_f32_e64 v2, s4, s6
-; GFX1064GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1064GISEL-NEXT: ; %bb.2:
-; GFX1064GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1064GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX1064GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1064GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064GISEL-NEXT: v_mul_f32_e32 v0, s2, v1
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1064GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX1064GISEL-NEXT: s_endpgm
;
; GFX1032DAGISEL-LABEL: uniform_value_half:
@@ -183,25 +165,20 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1032GISEL-LABEL: uniform_value_half:
; GFX1032GISEL: ; %bb.0: ; %entry
-; GFX1032GISEL-NEXT: s_clause 0x1
-; GFX1032GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032GISEL-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s0, s1
+; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032GISEL-NEXT: s_brev_b32 s3, 1
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032GISEL-NEXT: v_mul_f32_e32 v0, s2, v1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s4, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s4
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1032GISEL-NEXT: v_add_f32_e64 v2, s3, s5
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1032GISEL-NEXT: ; %bb.2:
-; GFX1032GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1032GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX1032GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX1032GISEL-NEXT: s_endpgm
;
; GFX1164DAGISEL-FAKE16-LABEL: uniform_value_half:
@@ -232,28 +209,22 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1164GISEL-FAKE16-LABEL: uniform_value_half:
; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-FAKE16-NEXT: s_brev_b32 s4, 1
; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-FAKE16-NEXT: v_add_f32_e64 v2, s4, s6
-; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, s2, v1
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-FAKE16-LABEL: uniform_value_half:
@@ -284,28 +255,22 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1132GISEL-FAKE16-LABEL: uniform_value_half:
; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX1132GISEL-FAKE16-NEXT: s_bcnt1_i32_b32 s0, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-FAKE16-NEXT: s_brev_b32 s3, 1
+; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132GISEL-FAKE16-NEXT: v_dual_mul_f32 v0, s2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-FAKE16-NEXT: v_add_f32_e64 v2, s3, s5
-; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1164DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -336,28 +301,22 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1164GISEL-TRUE16-LABEL: uniform_value_half:
; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-TRUE16-NEXT: s_brev_b32 s4, 1
; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-TRUE16-NEXT: v_add_f32_e64 v2, s4, s6
-; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s4
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, s2, v1
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-TRUE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -388,28 +347,22 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1132GISEL-TRUE16-LABEL: uniform_value_half:
; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX1132GISEL-TRUE16-NEXT: s_bcnt1_i32_b32 s0, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-TRUE16-NEXT: s_brev_b32 s3, 1
+; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, s2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-TRUE16-NEXT: v_add_f32_e64 v2, s3, s5
-; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s3
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-TRUE16-NEXT: s_endpgm
;
; GFX12DAGISEL-LABEL: uniform_value_half:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
index 31fd1123083eb..78e0dd4908f3f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.02 -global-isel=0 < %s | FileCheck -check-prefixes=GFX8DAGISEL %s
-; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX8GISEL %s
+; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 < %s | FileCheck -check-prefixes=GFX8GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX9DAGISEL %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX9GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GFX9GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10DAGISEL,GFX1064DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1064GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1064GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GFX10DAGISEL,GFX1032DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1032GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1032GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12DAGISEL %s
define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
@@ -44,23 +44,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX8GISEL-LABEL: uniform_value_half:
; GFX8GISEL: ; %bb.0: ; %entry
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8GISEL-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8GISEL-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s2
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX8GISEL-NEXT: v_readlane_b32 s6, v0, s5
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX8GISEL-NEXT: v_max_f32_e32 v1, s4, v1
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX8GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX8GISEL-NEXT: ; %bb.2:
-; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s4
; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX8GISEL-NEXT: flat_store_short v[0:1], v2
@@ -91,25 +81,15 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX9GISEL-LABEL: uniform_value_half:
; GFX9GISEL: ; %bb.0: ; %entry
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9GISEL-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX9GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX9GISEL-NEXT: v_mov_b32_e32 v2, s6
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX9GISEL-NEXT: v_max_f32_e32 v2, s4, v2
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX9GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX9GISEL-NEXT: ; %bb.2:
-; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX9GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX9GISEL-NEXT: s_endpgm
;
; GFX1064DAGISEL-LABEL: uniform_value_half:
@@ -135,28 +115,18 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1064DAGISEL-NEXT: global_store_short v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
-; GFX1064GISEL-LABEL: uniform_value_half:
-; GFX1064GISEL: ; %bb.0: ; %entry
-; GFX1064GISEL-NEXT: s_clause 0x1
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064GISEL-NEXT: s_mov_b32 s4, 0x7fc00000
-; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX1064GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1064GISEL-NEXT: v_max_f32_e64 v2, s4, s6
-; GFX1064GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1064GISEL-NEXT: ; %bb.2:
-; GFX1064GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1064GISEL-NEXT: global_store_short v0, v1, s[0:1]
-; GFX1064GISEL-NEXT: s_endpgm
+; GFX10GISEL-LABEL: uniform_value_half:
+; GFX10GISEL: ; %bb.0: ; %entry
+; GFX10GISEL-NEXT: s_clause 0x1
+; GFX10GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX10GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX10GISEL-NEXT: global_store_short v1, v0, s[0:1]
+; GFX10GISEL-NEXT: s_endpgm
;
; GFX1032DAGISEL-LABEL: uniform_value_half:
; GFX1032DAGISEL: ; %bb.0: ; %entry
@@ -181,29 +151,6 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1032DAGISEL-NEXT: global_store_short v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
-; GFX1032GISEL-LABEL: uniform_value_half:
-; GFX1032GISEL: ; %bb.0: ; %entry
-; GFX1032GISEL-NEXT: s_clause 0x1
-; GFX1032GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032GISEL-NEXT: s_mov_b32 s3, 0x7fc00000
-; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s4, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s4
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1032GISEL-NEXT: v_max_f32_e64 v2, s3, s5
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1032GISEL-NEXT: ; %bb.2:
-; GFX1032GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1032GISEL-NEXT: global_store_short v0, v1, s[0:1]
-; GFX1032GISEL-NEXT: s_endpgm
-;
; GFX1164DAGISEL-FAKE16-LABEL: uniform_value_half:
; GFX1164DAGISEL-FAKE16: ; %bb.0: ; %entry
; GFX1164DAGISEL-FAKE16-NEXT: s_clause 0x1
@@ -235,25 +182,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-FAKE16-NEXT: v_max_f32_e64 v2, s4, s6
-; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-FAKE16-LABEL: uniform_value_half:
@@ -287,25 +222,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x7fc00000
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-FAKE16-NEXT: v_max_f32_e64 v2, s3, s5
-; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1164DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -339,25 +262,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-TRUE16-NEXT: v_max_f32_e64 v2, s4, s6
-; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s4
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-TRUE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -391,25 +302,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x7fc00000
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-TRUE16-NEXT: v_max_f32_e64 v2, s3, s5
-; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s3
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-TRUE16-NEXT: s_endpgm
;
; GFX12DAGISEL-LABEL: uniform_value_half:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
index 21e3c110f2fb7..e45588dac7ff8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.02 -global-isel=0 < %s | FileCheck -check-prefixes=GFX8DAGISEL %s
-; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX8GISEL %s
+; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 < %s | FileCheck -check-prefixes=GFX8GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX9DAGISEL %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX9GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GFX9GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10DAGISEL,GFX1064DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1064GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1064GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GFX10DAGISEL,GFX1032DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1032GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GFX10GISEL,GFX1032GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12DAGISEL %s
define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
@@ -44,23 +44,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX8GISEL-LABEL: uniform_value_half:
; GFX8GISEL: ; %bb.0: ; %entry
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8GISEL-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8GISEL-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s2
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX8GISEL-NEXT: v_readlane_b32 s6, v0, s5
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX8GISEL-NEXT: v_min_f32_e32 v1, s4, v1
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX8GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX8GISEL-NEXT: ; %bb.2:
-; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s4
; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX8GISEL-NEXT: flat_store_short v[0:1], v2
@@ -91,25 +81,15 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX9GISEL-LABEL: uniform_value_half:
; GFX9GISEL: ; %bb.0: ; %entry
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9GISEL-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX9GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX9GISEL-NEXT: v_mov_b32_e32 v2, s6
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX9GISEL-NEXT: v_min_f32_e32 v2, s4, v2
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX9GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX9GISEL-NEXT: ; %bb.2:
-; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX9GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX9GISEL-NEXT: s_endpgm
;
; GFX1064DAGISEL-LABEL: uniform_value_half:
@@ -135,28 +115,18 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1064DAGISEL-NEXT: global_store_short v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
-; GFX1064GISEL-LABEL: uniform_value_half:
-; GFX1064GISEL: ; %bb.0: ; %entry
-; GFX1064GISEL-NEXT: s_clause 0x1
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064GISEL-NEXT: s_mov_b32 s4, 0x7fc00000
-; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX1064GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1064GISEL-NEXT: v_min_f32_e64 v2, s4, s6
-; GFX1064GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1064GISEL-NEXT: ; %bb.2:
-; GFX1064GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1064GISEL-NEXT: global_store_short v0, v1, s[0:1]
-; GFX1064GISEL-NEXT: s_endpgm
+; GFX10GISEL-LABEL: uniform_value_half:
+; GFX10GISEL: ; %bb.0: ; %entry
+; GFX10GISEL-NEXT: s_clause 0x1
+; GFX10GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX10GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX10GISEL-NEXT: global_store_short v1, v0, s[0:1]
+; GFX10GISEL-NEXT: s_endpgm
;
; GFX1032DAGISEL-LABEL: uniform_value_half:
; GFX1032DAGISEL: ; %bb.0: ; %entry
@@ -181,29 +151,6 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1032DAGISEL-NEXT: global_store_short v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
-; GFX1032GISEL-LABEL: uniform_value_half:
-; GFX1032GISEL: ; %bb.0: ; %entry
-; GFX1032GISEL-NEXT: s_clause 0x1
-; GFX1032GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032GISEL-NEXT: s_mov_b32 s3, 0x7fc00000
-; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s4, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s4
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1032GISEL-NEXT: v_min_f32_e64 v2, s3, s5
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1032GISEL-NEXT: ; %bb.2:
-; GFX1032GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1032GISEL-NEXT: global_store_short v0, v1, s[0:1]
-; GFX1032GISEL-NEXT: s_endpgm
-;
; GFX1164DAGISEL-FAKE16-LABEL: uniform_value_half:
; GFX1164DAGISEL-FAKE16: ; %bb.0: ; %entry
; GFX1164DAGISEL-FAKE16-NEXT: s_clause 0x1
@@ -235,25 +182,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-FAKE16-NEXT: v_min_f32_e64 v2, s4, s6
-; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-FAKE16-LABEL: uniform_value_half:
@@ -287,25 +222,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x7fc00000
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-FAKE16-NEXT: v_min_f32_e64 v2, s3, s5
-; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1164DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -339,25 +262,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-TRUE16-NEXT: v_min_f32_e64 v2, s4, s6
-; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s4
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-TRUE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -391,25 +302,13 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x7fc00000
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-TRUE16-NEXT: v_min_f32_e64 v2, s3, s5
-; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s3
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-TRUE16-NEXT: s_endpgm
;
; GFX12DAGISEL-LABEL: uniform_value_half:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
index feb61672253b5..ede0a21ed57fd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.02 -global-isel=0 < %s | FileCheck -check-prefixes=GFX8DAGISEL %s
-; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX8GISEL %s
+; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 < %s | FileCheck -check-prefixes=GFX8GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX9DAGISEL %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX9GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GFX9GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1064DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1064GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1064GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1032DAGISEL %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1032GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1032GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-FAKE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164DAGISEL,GFX1164DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1164GISEL,GFX1164GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=0 < %s | FileCheck -check-prefixes=GFX1132DAGISEL,GFX1132DAGISEL-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -global-isel=1 < %s | FileCheck -check-prefixes=GFX1132GISEL,GFX1132GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12DAGISEL %s
define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
@@ -45,22 +45,16 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX8GISEL-LABEL: uniform_value_half:
; GFX8GISEL: ; %bb.0: ; %entry
; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8GISEL-NEXT: s_mov_b32 s4, 0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX8GISEL-NEXT: v_readlane_b32 s6, v0, s5
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX8GISEL-NEXT: v_sub_f32_e32 v1, s4, v1
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX8GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX8GISEL-NEXT: ; %bb.2:
-; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s4
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_cvt_f16_f32_e32 v2, s2
; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX8GISEL-NEXT: flat_store_short v[0:1], v2
@@ -92,24 +86,18 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
; GFX9GISEL-LABEL: uniform_value_half:
; GFX9GISEL: ; %bb.0: ; %entry
; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9GISEL-NEXT: s_mov_b32 s4, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX9GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX9GISEL-NEXT: v_mov_b32_e32 v2, s6
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX9GISEL-NEXT: v_sub_f32_e32 v2, s4, v2
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX9GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX9GISEL-NEXT: ; %bb.2:
-; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX9GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX9GISEL-NEXT: s_endpgm
;
; GFX1064DAGISEL-LABEL: uniform_value_half:
@@ -137,25 +125,19 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1064GISEL-LABEL: uniform_value_half:
; GFX1064GISEL: ; %bb.0: ; %entry
-; GFX1064GISEL-NEXT: s_clause 0x1
; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064GISEL-NEXT: s_mov_b32 s4, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s5, s[2:3]
-; GFX1064GISEL-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1064GISEL-NEXT: v_sub_f32_e64 v2, s4, s6
-; GFX1064GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1064GISEL-NEXT: ; %bb.2:
-; GFX1064GISEL-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1064GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX1064GISEL-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1064GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064GISEL-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1064GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX1064GISEL-NEXT: s_endpgm
;
; GFX1032DAGISEL-LABEL: uniform_value_half:
@@ -183,25 +165,20 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1032GISEL-LABEL: uniform_value_half:
; GFX1032GISEL: ; %bb.0: ; %entry
-; GFX1032GISEL-NEXT: s_clause 0x1
-; GFX1032GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032GISEL-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s0, s1
+; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032GISEL-NEXT: s_mov_b32 s3, 0
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032GISEL-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s4, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s4
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1032GISEL-NEXT: v_sub_f32_e64 v2, s3, s5
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1032GISEL-NEXT: ; %bb.2:
-; GFX1032GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1032GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX1032GISEL-NEXT: global_store_short v1, v0, s[0:1]
; GFX1032GISEL-NEXT: s_endpgm
;
; GFX1164DAGISEL-FAKE16-LABEL: uniform_value_half:
@@ -232,28 +209,22 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1164GISEL-FAKE16-LABEL: uniform_value_half:
; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-FAKE16-NEXT: v_sub_f32_e64 v2, s4, s6
-; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s4
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-FAKE16-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-FAKE16-LABEL: uniform_value_half:
@@ -284,28 +255,23 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1132GISEL-FAKE16-LABEL: uniform_value_half:
; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX1132GISEL-FAKE16-NEXT: s_bcnt1_i32_b32 s0, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s3, 0
+; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132GISEL-FAKE16-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-FAKE16-NEXT: v_sub_f32_e64 v2, s3, s5
-; GFX1132GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
-; GFX1132GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s3
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1164DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -336,28 +302,22 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1164GISEL-TRUE16-LABEL: uniform_value_half:
; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s5, s[2:3]
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s6, v1, s5
-; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[2:3], s5
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX1164GISEL-TRUE16-NEXT: v_sub_f32_e64 v2, s4, s6
-; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s4
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-TRUE16-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1164GISEL-TRUE16-NEXT: s_endpgm
;
; GFX1132DAGISEL-TRUE16-LABEL: uniform_value_half:
@@ -388,28 +348,23 @@ define amdgpu_kernel void @uniform_value_half(ptr addrspace(1) %out, half %in) {
;
; GFX1132GISEL-TRUE16-LABEL: uniform_value_half:
; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX1132GISEL-TRUE16-NEXT: s_bcnt1_i32_b32 s0, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, s0
; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s3, 0
+; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132GISEL-TRUE16-NEXT: v_mul_f32_e64 v0, -s2, v1
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s2
; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s4, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s5, v1, s4
-; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s2, s4
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1132GISEL-TRUE16-NEXT: v_sub_f32_e64 v2, s3, s5
-; GFX1132GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB0_1
-; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
-; GFX1132GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s3
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1132GISEL-TRUE16-NEXT: s_endpgm
;
; GFX12DAGISEL-LABEL: uniform_value_half:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index 265cb0af5f6be..916c0a2e87e9f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -2,11 +2,11 @@
; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck --check-prefix=VARIANT0 %s
; RUN: llc -mtriple=amdgpu6.00 -mattr=+auto-waitcnt-before-barrier < %s | FileCheck --check-prefix=VARIANT1 %s
; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=VARIANT2 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=VARIANT2-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=VARIANT2-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -mattr=+auto-waitcnt-before-barrier < %s | FileCheck --check-prefix=VARIANT3 %s
; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck --check-prefix=VARIANT4 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=+auto-waitcnt-before-barrier < %s | FileCheck --check-prefix=VARIANT5 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck --check-prefix=VARIANT6 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck --check-prefix=VARIANT6 %s
define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
; VARIANT0-LABEL: test_barrier:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll
index bd8a2a68356ac..ab531ca3b4447 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11-SDAG-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11-SDAG-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11-GISEL-FAKE16 %s
define half @v_sqrt_f16(half %src) {
; GCN-LABEL: v_sqrt_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
index 928cb355ccea5..94668c0c126d9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
@@ -7,8 +7,8 @@
; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11-PACKED,GFX11-PACKED-FAKE16 %s
; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -mattr=+real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-SDAG,GFX12-PACKED-SDAG-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-SDAG,GFX12-PACKED-SDAG-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdhsa -mattr=+real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdhsa -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdhsa -mattr=+real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdhsa -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12-PACKED,GFX12-PACKED-GISEL,GFX12-PACKED-GISEL-FAKE16 %s
define amdgpu_kernel void @tbuffer_store_d16_x(<4 x i32> %rsrc, half %data, i32 %vindex) {
; PREGFX10-UNPACKED-LABEL: tbuffer_store_d16_x:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll
index c4e8ae2313ef2..cbdf602bdd9e4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll
@@ -1,8 +1,8 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu9.0a < %s | FileCheck --check-prefixes=GCN,GFX90A,GFX90A-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.0a < %s | FileCheck --check-prefixes=GCN,GFX90A,GFX90A-GISEL %s
+; xUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.0a < %s | FileCheck --check-prefixes=GCN,GFX90A,GFX90A-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.4 --amdhsa-code-object-version=6 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
+; xUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
+; xUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.4 --amdhsa-code-object-version=6 < %s | FileCheck --check-prefixes=GCN,GFX942 %s
; DPP control value 337 is valid for 64-bit DPP on gfx942
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
index 909f4c7c2c593..43070786bb022 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
@@ -3,8 +3,8 @@
; RUN: llc -mtriple=amdgpu10.10--amdhsa < %s | FileCheck -check-prefixes=GFX1010-SDAG %s
; RUN: llc -mtriple=amdgpu11.00--amdhsa -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX1100-SDAG %s
-; RUN: llc -mtriple=amdgpu8.02--amdhsa -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX802-GISEL %s
-; RUN: llc -mtriple=amdgpu10.10--amdhsa -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1010-GISEL %s
+; xUN: llc -mtriple=amdgpu8.02--amdhsa -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX802-GISEL %s
+; xUN: llc -mtriple=amdgpu10.10--amdhsa -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1010-GISEL %s
; RUN: llc -mtriple=amdgpu11.00--amdhsa -amdgpu-enable-vopd=0 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX1100-GISEL %s
declare i32 @llvm.amdgcn.writelane(i32, i32, i32) #0
@@ -2085,11 +2085,11 @@ define void @test_writelane_half(ptr addrspace(1) %out, half %src, i32 %src1) {
; GFX802-GISEL: ; %bb.0:
; GFX802-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX802-GISEL-NEXT: flat_load_ushort v4, v[0:1]
-; GFX802-GISEL-NEXT: v_readfirstlane_b32 s4, v3
-; GFX802-GISEL-NEXT: v_readfirstlane_b32 s5, v2
-; GFX802-GISEL-NEXT: s_mov_b32 m0, s4
+; GFX802-GISEL-NEXT: v_readfirstlane_b32 s5, v3
+; GFX802-GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GFX802-GISEL-NEXT: s_mov_b32 m0, s5
; GFX802-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX802-GISEL-NEXT: v_writelane_b32 v4, s5, m0
+; GFX802-GISEL-NEXT: v_writelane_b32 v4, s4, m0
; GFX802-GISEL-NEXT: flat_store_short v[0:1], v4
; GFX802-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX802-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2241,11 +2241,11 @@ define void @test_writelane_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1
; GFX802-GISEL: ; %bb.0:
; GFX802-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX802-GISEL-NEXT: flat_load_ushort v4, v[0:1]
-; GFX802-GISEL-NEXT: v_readfirstlane_b32 s4, v3
-; GFX802-GISEL-NEXT: v_readfirstlane_b32 s5, v2
-; GFX802-GISEL-NEXT: s_mov_b32 m0, s4
+; GFX802-GISEL-NEXT: v_readfirstlane_b32 s5, v3
+; GFX802-GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GFX802-GISEL-NEXT: s_mov_b32 m0, s5
; GFX802-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX802-GISEL-NEXT: v_writelane_b32 v4, s5, m0
+; GFX802-GISEL-NEXT: v_writelane_b32 v4, s4, m0
; GFX802-GISEL-NEXT: flat_store_short v[0:1], v4
; GFX802-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX802-GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
index 186afbee62ebf..a2cbda17b41cd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
@@ -4893,53 +4893,101 @@ define float @v_exp_f32_from_fpext_f16(i16 %src.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_f32_from_fpext_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x3fb8aa3b
-; GFX900-NEXT: s_mov_b32 s5, 0x32a5705f
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; GFX900-NEXT: v_rndne_f32_e32 v2, v1
-; GFX900-NEXT: v_fma_f32 v3, v0, s4, -v1
-; GFX900-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_fma_f32 v3, v0, s5, v3
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX900-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: s_mov_b32 s4, 0xc2ce8ed0
-; GFX900-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x42b17218
-; GFX900-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; GFX900-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_f32_from_fpext_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3fb8aa3b
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x32a5705f
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GFX900-SDAG-NEXT: v_rndne_f32_e32 v2, v1
+; GFX900-SDAG-NEXT: v_fma_f32 v3, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_fma_f32 v3, v0, s5, v3
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0xc2ce8ed0
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x42b17218
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_f32_from_fpext_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x3fb8aa3b
-; SI-NEXT: s_mov_b32 s5, 0x32a5705f
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; SI-NEXT: v_rndne_f32_e32 v2, v1
-; SI-NEXT: v_fma_f32 v3, v0, s4, -v1
-; SI-NEXT: v_sub_f32_e32 v1, v1, v2
-; SI-NEXT: v_fma_f32 v3, v0, s5, v3
-; SI-NEXT: v_add_f32_e32 v1, v1, v3
-; SI-NEXT: v_cvt_i32_f32_e32 v2, v2
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: s_mov_b32 s4, 0xc2ce8ed0
-; SI-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; SI-NEXT: s_mov_b32 s4, 0x42b17218
-; SI-NEXT: v_ldexp_f32_e32 v1, v1, v2
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SI-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-LABEL: v_exp_f32_from_fpext_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fb8aa3b
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x32a5705f
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; GFX900-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v4
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x42b17218
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-SDAG-LABEL: v_exp_f32_from_fpext_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3fb8aa3b
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x32a5705f
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_rndne_f32_e32 v2, v1
+; SI-SDAG-NEXT: v_fma_f32 v3, v0, s4, -v1
+; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_fma_f32 v3, v0, s5, v3
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xc2ce8ed0
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x42b17218
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f32_from_fpext_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fb8aa3b
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x32a5705f
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; SI-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; SI-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v4
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v2
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x42b17218
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; SI-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_f32_from_fpext_f16:
; R600: ; %bb.0:
@@ -4985,58 +5033,111 @@ define float @v_exp_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_f32_from_fpext_math_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x3fb8aa3b
-; GFX900-NEXT: s_mov_b32 s5, 0x32a5705f
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_rndne_f32_e32 v3, v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_sub_f32_e32 v1, v1, v3
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: s_mov_b32 s4, 0xc2ce8ed0
-; GFX900-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x42b17218
-; GFX900-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; GFX900-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_f32_from_fpext_math_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3fb8aa3b
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x32a5705f
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0xc2ce8ed0
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x42b17218
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_f32_from_fpext_math_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x3fb8aa3b
-; SI-NEXT: s_mov_b32 s5, 0x32a5705f
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_rndne_f32_e32 v3, v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_sub_f32_e32 v1, v1, v3
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cvt_i32_f32_e32 v3, v3
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: s_mov_b32 s4, 0xc2ce8ed0
-; SI-NEXT: s_mov_b32 s5, 0x42b17218
-; SI-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; SI-NEXT: v_ldexp_f32_e32 v1, v1, v3
-; SI-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SI-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-LABEL: v_exp_f32_from_fpext_math_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fb8aa3b
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x32a5705f
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_rndne_f32_e32 v2, v3
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0xc2ce8ed0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x42b17218
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-SDAG-LABEL: v_exp_f32_from_fpext_math_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3fb8aa3b
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x32a5705f
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xc2ce8ed0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x42b17218
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f32_from_fpext_math_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x32a5705f
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x7f800000
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fb8aa3b
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; SI-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; SI-GISEL-NEXT: v_cvt_i32_f32_e32 v3, v4
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xc2ce8ed0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x42b17218
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; SI-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_f32_from_fpext_math_f16:
; R600: ; %bb.0:
@@ -5154,17 +5255,29 @@ define float @v_exp_f32_from_fpext_math_f16_fast(i16 %src0.i, i16 %src1.i) {
; GCN-NEXT: v_exp_f32_e32 v0, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_f32_from_fpext_math_f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_f32_from_fpext_math_f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f32_from_fpext_math_f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_f32_from_fpext_math_f16_fast:
; R600: ; %bb.0:
@@ -5212,58 +5325,111 @@ define float @v_exp_f32_from_fpext_math_f16_daz(i16 %src0.i, i16 %src1.i) #0 {
; VI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_f32_from_fpext_math_f16_daz:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x3fb8aa3b
-; GFX900-NEXT: s_mov_b32 s5, 0x32a5705f
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_rndne_f32_e32 v3, v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_sub_f32_e32 v1, v1, v3
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: s_mov_b32 s4, 0xc2ce8ed0
-; GFX900-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x42b17218
-; GFX900-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; GFX900-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_f32_from_fpext_math_f16_daz:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3fb8aa3b
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x32a5705f
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0xc2ce8ed0
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x42b17218
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_f32_from_fpext_math_f16_daz:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x3fb8aa3b
-; SI-NEXT: s_mov_b32 s5, 0x32a5705f
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_rndne_f32_e32 v3, v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_sub_f32_e32 v1, v1, v3
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cvt_i32_f32_e32 v3, v3
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: s_mov_b32 s4, 0xc2ce8ed0
-; SI-NEXT: s_mov_b32 s5, 0x42b17218
-; SI-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; SI-NEXT: v_ldexp_f32_e32 v1, v1, v3
-; SI-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SI-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-LABEL: v_exp_f32_from_fpext_math_f16_daz:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fb8aa3b
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x32a5705f
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_rndne_f32_e32 v2, v3
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0xc2ce8ed0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x42b17218
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-SDAG-LABEL: v_exp_f32_from_fpext_math_f16_daz:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3fb8aa3b
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x32a5705f
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xc2ce8ed0
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x42b17218
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_f32_from_fpext_math_f16_daz:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x32a5705f
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x7f800000
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fb8aa3b
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; SI-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; SI-GISEL-NEXT: v_cvt_i32_f32_e32 v3, v4
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xc2ce8ed0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x42b17218
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; SI-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_f32_from_fpext_math_f16_daz:
; R600: ; %bb.0:
@@ -5349,14 +5515,23 @@ define half @v_exp_fabs_f16(half %in) {
}
define half @v_exp_fneg_fabs_f16(half %in) {
-; GCN-LABEL: v_exp_fneg_fabs_f16:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GCN-NEXT: v_mul_f32_e32 v0, 0xbfb8aa3b, v0
-; GCN-NEXT: v_exp_f32_e32 v0, v0
-; GCN-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GCN-SDAG-LABEL: v_exp_fneg_fabs_f16:
+; GCN-SDAG: ; %bb.0:
+; GCN-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; GCN-SDAG-NEXT: v_mul_f32_e32 v0, 0xbfb8aa3b, v0
+; GCN-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GCN-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-GISEL-LABEL: v_exp_fneg_fabs_f16:
+; GCN-GISEL: ; %bb.0:
+; GCN-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-GISEL-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; GCN-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GCN-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GCN-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_fneg_fabs_f16:
; SI: ; %bb.0:
@@ -5383,14 +5558,23 @@ define half @v_exp_fneg_fabs_f16(half %in) {
}
define half @v_exp_fneg_f16(half %in) {
-; GCN-LABEL: v_exp_fneg_f16:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GCN-NEXT: v_mul_f32_e32 v0, 0xbfb8aa3b, v0
-; GCN-NEXT: v_exp_f32_e32 v0, v0
-; GCN-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GCN-SDAG-LABEL: v_exp_fneg_f16:
+; GCN-SDAG: ; %bb.0:
+; GCN-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GCN-SDAG-NEXT: v_mul_f32_e32 v0, 0xbfb8aa3b, v0
+; GCN-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GCN-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-GISEL-LABEL: v_exp_fneg_f16:
+; GCN-GISEL: ; %bb.0:
+; GCN-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-GISEL-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GCN-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GCN-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GCN-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_fneg_f16:
; SI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
index 1514adbfe61ad..260274c90a54a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
@@ -4955,53 +4955,101 @@ define float @v_exp10_f32_from_fpext_f16(i16 %src.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_f32_from_fpext_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x40549a78
-; GFX900-NEXT: s_mov_b32 s5, 0x33979a37
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; GFX900-NEXT: v_rndne_f32_e32 v2, v1
-; GFX900-NEXT: v_fma_f32 v3, v0, s4, -v1
-; GFX900-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_fma_f32 v3, v0, s5, v3
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX900-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: s_mov_b32 s4, 0xc23369f4
-; GFX900-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x421a209b
-; GFX900-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; GFX900-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_f32_from_fpext_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x40549a78
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x33979a37
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; GFX900-SDAG-NEXT: v_rndne_f32_e32 v2, v1
+; GFX900-SDAG-NEXT: v_fma_f32 v3, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_fma_f32 v3, v0, s5, v3
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0xc23369f4
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x421a209b
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_f32_from_fpext_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x40549a78
-; SI-NEXT: s_mov_b32 s5, 0x33979a37
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; SI-NEXT: v_rndne_f32_e32 v2, v1
-; SI-NEXT: v_fma_f32 v3, v0, s4, -v1
-; SI-NEXT: v_sub_f32_e32 v1, v1, v2
-; SI-NEXT: v_fma_f32 v3, v0, s5, v3
-; SI-NEXT: v_add_f32_e32 v1, v1, v3
-; SI-NEXT: v_cvt_i32_f32_e32 v2, v2
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: s_mov_b32 s4, 0xc23369f4
-; SI-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; SI-NEXT: s_mov_b32 s4, 0x421a209b
-; SI-NEXT: v_ldexp_f32_e32 v1, v1, v2
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SI-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-LABEL: v_exp10_f32_from_fpext_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x40549a78
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x33979a37
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x40549a78, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; GFX900-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v4
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0xc23369f4
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x421a209b
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-SDAG-LABEL: v_exp10_f32_from_fpext_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x40549a78
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x33979a37
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; SI-SDAG-NEXT: v_rndne_f32_e32 v2, v1
+; SI-SDAG-NEXT: v_fma_f32 v3, v0, s4, -v1
+; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_fma_f32 v3, v0, s5, v3
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xc23369f4
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x421a209b
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f32_from_fpext_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x40549a78
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x33979a37
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x40549a78, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; SI-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; SI-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v4
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0xc23369f4
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v2
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x421a209b
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; SI-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_f32_from_fpext_f16:
; R600: ; %bb.0:
@@ -5047,58 +5095,111 @@ define float @v_exp10_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_f32_from_fpext_math_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x40549a78
-; GFX900-NEXT: s_mov_b32 s5, 0x33979a37
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_rndne_f32_e32 v3, v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_sub_f32_e32 v1, v1, v3
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: s_mov_b32 s4, 0xc23369f4
-; GFX900-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x421a209b
-; GFX900-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; GFX900-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_f32_from_fpext_math_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x40549a78
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x33979a37
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0xc23369f4
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x421a209b
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_f32_from_fpext_math_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x40549a78
-; SI-NEXT: s_mov_b32 s5, 0x33979a37
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_rndne_f32_e32 v3, v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_sub_f32_e32 v1, v1, v3
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cvt_i32_f32_e32 v3, v3
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: s_mov_b32 s4, 0xc23369f4
-; SI-NEXT: s_mov_b32 s5, 0x421a209b
-; SI-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; SI-NEXT: v_ldexp_f32_e32 v1, v1, v3
-; SI-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SI-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-LABEL: v_exp10_f32_from_fpext_math_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x40549a78
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x33979a37
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x40549a78, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_rndne_f32_e32 v2, v3
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0xc23369f4
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x421a209b
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-SDAG-LABEL: v_exp10_f32_from_fpext_math_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x40549a78
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x33979a37
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xc23369f4
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x421a209b
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f32_from_fpext_math_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x33979a37
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x7f800000
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x40549a78
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x40549a78, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; SI-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; SI-GISEL-NEXT: v_cvt_i32_f32_e32 v3, v4
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xc23369f4
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x421a209b
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; SI-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_f32_from_fpext_math_f16:
; R600: ; %bb.0:
@@ -5219,20 +5320,35 @@ define float @v_exp10_f32_from_fpext_math_f16_fast(i16 %src0.i, i16 %src1.i) {
; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_f32_from_fpext_math_f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3a2784bc, v0
-; SI-NEXT: v_mul_f32_e32 v0, 0x40549000, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_f32_from_fpext_math_f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3a2784bc, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549000, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f32_from_fpext_math_f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3a2784bc, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549000, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_f32_from_fpext_math_f16_fast:
; R600: ; %bb.0:
@@ -5280,58 +5396,111 @@ define float @v_exp10_f32_from_fpext_math_f16_daz(i16 %src0.i, i16 %src1.i) #0 {
; VI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_f32_from_fpext_math_f16_daz:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x40549a78
-; GFX900-NEXT: s_mov_b32 s5, 0x33979a37
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_rndne_f32_e32 v3, v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_sub_f32_e32 v1, v1, v3
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: s_mov_b32 s4, 0xc23369f4
-; GFX900-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x421a209b
-; GFX900-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; GFX900-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_f32_from_fpext_math_f16_daz:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x40549a78
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x33979a37
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0xc23369f4
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x421a209b
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_f32_from_fpext_math_f16_daz:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x40549a78
-; SI-NEXT: s_mov_b32 s5, 0x33979a37
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_rndne_f32_e32 v3, v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_sub_f32_e32 v1, v1, v3
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cvt_i32_f32_e32 v3, v3
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: s_mov_b32 s4, 0xc23369f4
-; SI-NEXT: s_mov_b32 s5, 0x421a209b
-; SI-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
-; SI-NEXT: v_ldexp_f32_e32 v1, v1, v3
-; SI-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SI-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; GFX900-GISEL-LABEL: v_exp10_f32_from_fpext_math_f16_daz:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x40549a78
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x33979a37
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x40549a78, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_rndne_f32_e32 v2, v3
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0xc23369f4
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x421a209b
+; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-SDAG-LABEL: v_exp10_f32_from_fpext_math_f16_daz:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x40549a78
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x33979a37
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xc23369f4
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x421a209b
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_f32_from_fpext_math_f16_daz:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x33979a37
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x7f800000
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x40549a78
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x40549a78, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
+; SI-GISEL-NEXT: v_rndne_f32_e32 v4, v3
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, v3, v4
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
+; SI-GISEL-NEXT: v_cvt_i32_f32_e32 v3, v4
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xc23369f4
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x421a209b
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
+; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; SI-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_f32_from_fpext_math_f16_daz:
; R600: ; %bb.0:
@@ -5417,14 +5586,23 @@ define half @v_exp10_fabs_f16(half %in) {
}
define half @v_exp10_fneg_fabs_f16(half %in) {
-; GCN-LABEL: v_exp10_fneg_fabs_f16:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GCN-NEXT: v_mul_f32_e32 v0, 0xc0549a78, v0
-; GCN-NEXT: v_exp_f32_e32 v0, v0
-; GCN-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GCN-SDAG-LABEL: v_exp10_fneg_fabs_f16:
+; GCN-SDAG: ; %bb.0:
+; GCN-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; GCN-SDAG-NEXT: v_mul_f32_e32 v0, 0xc0549a78, v0
+; GCN-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GCN-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-GISEL-LABEL: v_exp10_fneg_fabs_f16:
+; GCN-GISEL: ; %bb.0:
+; GCN-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-GISEL-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; GCN-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GCN-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GCN-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_fneg_fabs_f16:
; SI: ; %bb.0:
@@ -5451,14 +5629,23 @@ define half @v_exp10_fneg_fabs_f16(half %in) {
}
define half @v_exp10_fneg_f16(half %in) {
-; GCN-LABEL: v_exp10_fneg_f16:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GCN-NEXT: v_mul_f32_e32 v0, 0xc0549a78, v0
-; GCN-NEXT: v_exp_f32_e32 v0, v0
-; GCN-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GCN-SDAG-LABEL: v_exp10_fneg_f16:
+; GCN-SDAG: ; %bb.0:
+; GCN-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GCN-SDAG-NEXT: v_mul_f32_e32 v0, 0xc0549a78, v0
+; GCN-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GCN-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-GISEL-LABEL: v_exp10_fneg_f16:
+; GCN-GISEL: ; %bb.0:
+; GCN-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-GISEL-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GCN-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GCN-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GCN-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_fneg_f16:
; SI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 2f5dc77d904d5..5148489812151 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -2781,16 +2781,27 @@ define float @v_exp2_f32_from_fpext_f16(i16 %src.i) {
}
define float @v_exp2_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
-; SI-LABEL: v_exp2_f32_from_fpext_math_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_f32_from_fpext_math_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp2_f32_from_fpext_math_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_exp2_f32_from_fpext_math_f16:
; VI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index c1dd80d405a12..4e8f11cf1aa1b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -2,10 +2,10 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=CHECK,SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=CHECK,SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=CHECK,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=CHECK,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=CHECK,GFX11-GISEL %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
define amdgpu_gs half @v_fptrunc_round_f32_to_f16_tonearest(float %a) {
; SDAG-LABEL: v_fptrunc_round_f32_to_f16_tonearest:
@@ -514,9 +514,9 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addr
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_f32_to_f16_upward:
@@ -524,8 +524,8 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addr
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: s_and_b32 s0, s0, 0xffff
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_f32_to_f16_upward:
@@ -564,9 +564,9 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_downward(float inreg %a, ptr ad
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_f32_to_f16_downward:
@@ -574,8 +574,8 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_downward(float inreg %a, ptr ad
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: s_and_b32 s0, s0, 0xffff
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_f32_to_f16_downward:
@@ -609,17 +609,17 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_towardzero(float inreg %a, ptr
;
; GFX11-GISEL-LABEL: s_fptrunc_round_f32_to_f16_towardzero:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: v_cvt_pk_rtz_f16_f32_e64 v0, s0, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-NEXT: v_cvt_pk_rtz_f16_f32_e32 v0, s0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_f32_to_f16_towardzero:
; GISEL: ; %bb.0:
; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, s0, v0
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: s_and_b32 s0, s0, 0xffff
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_f32_to_f16_towardzero:
@@ -968,11 +968,10 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> in
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, s0
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, s1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-GISEL-NEXT: s_and_b32 s1, 0xffff, s1
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward:
@@ -1036,11 +1035,10 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, s0
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, s1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-GISEL-NEXT: s_and_b32 s1, 0xffff, s1
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_downward:
@@ -1096,14 +1094,13 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_towardzero(<2 x float
;
; GFX11-GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_towardzero:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: v_cvt_pk_rtz_f16_f32_e64 v0, s0, s0
-; GFX11-GISEL-NEXT: v_cvt_pk_rtz_f16_f32_e64 v1, s1, s0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_cvt_pk_rtz_f16_f32_e32 v0, s0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_cvt_pk_rtz_f16_f32_e32 v1, s1, v0
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-GISEL-NEXT: s_and_b32 s1, 0xffff, s1
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_towardzero:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll b/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll
index 24dbca145e81c..ac86b35e7eb28 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.get.rounding.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GFX678-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GFX678-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GFX678-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GFX678-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GFX678-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GFX678-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX678-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX678-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX678-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX10 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11 %s
declare i32 @llvm.get.rounding()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
index 20b4e0ac2e710..6f73787cdf03d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -9,67 +9,123 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.31 < %s | FileCheck --check-prefixes=GFX10CHECK,GFX10GLISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11SELDAG,GFX11SELDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11SELDAG,GFX11SELDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11GLISEL,GFX11GLISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11GLISEL,GFX11GLISEL-TRUE16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11GLISEL,GFX11GLISEL-FAKE16 %s
; FIXME: There are code size regressions in GlobalISel due to use of SGPRs and
; moving those SGPRs into VGPRs.
define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
-; GFX7CHECK-LABEL: sgpr_isnan_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7CHECK-NEXT: s_mov_b32 s3, 0xf000
-; GFX7CHECK-NEXT: s_mov_b32 s2, -1
-; GFX7CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7CHECK-NEXT: s_and_b32 s4, s6, 0x7fff
-; GFX7CHECK-NEXT: s_cmpk_gt_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: s_cselect_b32 s4, -1, 0
-; GFX7CHECK-NEXT: v_mov_b32_e32 v0, s4
-; GFX7CHECK-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GFX7CHECK-NEXT: s_endpgm
-;
-; GFX8CHECK-LABEL: sgpr_isnan_f16:
-; GFX8CHECK: ; %bb.0:
-; GFX8CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8CHECK-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
-; GFX8CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX8CHECK-NEXT: s_cselect_b32 s2, -1, 0
-; GFX8CHECK-NEXT: v_mov_b32_e32 v0, s0
-; GFX8CHECK-NEXT: v_mov_b32_e32 v1, s1
-; GFX8CHECK-NEXT: v_mov_b32_e32 v2, s2
-; GFX8CHECK-NEXT: flat_store_dword v[0:1], v2
-; GFX8CHECK-NEXT: s_endpgm
-;
-; GFX9CHECK-LABEL: sgpr_isnan_f16:
-; GFX9CHECK: ; %bb.0:
-; GFX9CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9CHECK-NEXT: v_mov_b32_e32 v0, 0
-; GFX9CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9CHECK-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
-; GFX9CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9CHECK-NEXT: s_cselect_b32 s2, -1, 0
-; GFX9CHECK-NEXT: v_mov_b32_e32 v1, s2
-; GFX9CHECK-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX9CHECK-NEXT: s_endpgm
-;
-; GFX10CHECK-LABEL: sgpr_isnan_f16:
-; GFX10CHECK: ; %bb.0:
-; GFX10CHECK-NEXT: s_clause 0x1
-; GFX10CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10CHECK-NEXT: v_mov_b32_e32 v0, 0
-; GFX10CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10CHECK-NEXT: v_cmp_class_f16_e64 s2, s2, 3
-; GFX10CHECK-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX10CHECK-NEXT: s_cselect_b32 s2, -1, 0
-; GFX10CHECK-NEXT: v_mov_b32_e32 v1, s2
-; GFX10CHECK-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX10CHECK-NEXT: s_endpgm
+; GFX7SELDAG-LABEL: sgpr_isnan_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_load_dword s6, s[4:5], 0xb
+; GFX7SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7SELDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX7SELDAG-NEXT: s_mov_b32 s2, -1
+; GFX7SELDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7SELDAG-NEXT: s_and_b32 s4, s6, 0x7fff
+; GFX7SELDAG-NEXT: s_cmpk_gt_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: s_cselect_b32 s4, -1, 0
+; GFX7SELDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX7SELDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7SELDAG-NEXT: s_endpgm
+;
+; GFX7GLISEL-LABEL: sgpr_isnan_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_load_dword s3, s[4:5], 0xb
+; GFX7GLISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7GLISEL-NEXT: s_mov_b32 s2, -1
+; GFX7GLISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7GLISEL-NEXT: s_and_b32 s3, s3, 0x7fff
+; GFX7GLISEL-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX7GLISEL-NEXT: s_cmpk_gt_u32 s3, 0x7c00
+; GFX7GLISEL-NEXT: s_cselect_b32 s3, -1, 0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX7GLISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX7GLISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7GLISEL-NEXT: s_endpgm
+;
+; GFX8SELDAG-LABEL: sgpr_isnan_f16:
+; GFX8SELDAG: ; %bb.0:
+; GFX8SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8SELDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8SELDAG-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
+; GFX8SELDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8SELDAG-NEXT: s_cselect_b32 s2, -1, 0
+; GFX8SELDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8SELDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX8SELDAG-NEXT: flat_store_dword v[0:1], v2
+; GFX8SELDAG-NEXT: s_endpgm
+;
+; GFX8GLISEL-LABEL: sgpr_isnan_f16:
+; GFX8GLISEL: ; %bb.0:
+; GFX8GLISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GLISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GLISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
+; GFX8GLISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GLISEL-NEXT: s_cselect_b32 s2, -1, 0
+; GFX8GLISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8GLISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8GLISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8GLISEL-NEXT: flat_store_dword v[0:1], v2
+; GFX8GLISEL-NEXT: s_endpgm
+;
+; GFX9SELDAG-LABEL: sgpr_isnan_f16:
+; GFX9SELDAG: ; %bb.0:
+; GFX9SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9SELDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9SELDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9SELDAG-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
+; GFX9SELDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9SELDAG-NEXT: s_cselect_b32 s2, -1, 0
+; GFX9SELDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX9SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9SELDAG-NEXT: s_endpgm
+;
+; GFX9GLISEL-LABEL: sgpr_isnan_f16:
+; GFX9GLISEL: ; %bb.0:
+; GFX9GLISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GLISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9GLISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9GLISEL-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
+; GFX9GLISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GLISEL-NEXT: s_cselect_b32 s2, -1, 0
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9GLISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9GLISEL-NEXT: s_endpgm
+;
+; GFX10SELDAG-LABEL: sgpr_isnan_f16:
+; GFX10SELDAG: ; %bb.0:
+; GFX10SELDAG-NEXT: s_clause 0x1
+; GFX10SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10SELDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX10SELDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10SELDAG-NEXT: v_cmp_class_f16_e64 s2, s2, 3
+; GFX10SELDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10SELDAG-NEXT: s_cselect_b32 s2, -1, 0
+; GFX10SELDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX10SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10SELDAG-NEXT: s_endpgm
+;
+; GFX10GLISEL-LABEL: sgpr_isnan_f16:
+; GFX10GLISEL: ; %bb.0:
+; GFX10GLISEL-NEXT: s_clause 0x1
+; GFX10GLISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10GLISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10GLISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10GLISEL-NEXT: v_cmp_class_f16_e64 s2, s2, 3
+; GFX10GLISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX10GLISEL-NEXT: s_cselect_b32 s2, -1, 0
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX10GLISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10GLISEL-NEXT: s_endpgm
;
; GFX11SELDAG-TRUE16-LABEL: sgpr_isnan_f16:
; GFX11SELDAG-TRUE16: ; %bb.0:
@@ -104,14 +160,13 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX11GLISEL-TRUE16-NEXT: s_clause 0x1
; GFX11GLISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX11GLISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 3
+; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v0, 3 :: v_dual_mov_b32 v1, 0
; GFX11GLISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, s2, v0.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11GLISEL-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11GLISEL-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11GLISEL-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11GLISEL-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11GLISEL-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11GLISEL-TRUE16-NEXT: s_endpgm
;
; GFX11GLISEL-FAKE16-LABEL: sgpr_isnan_f16:
@@ -119,12 +174,13 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX11GLISEL-FAKE16-NEXT: s_clause 0x1
; GFX11GLISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX11GLISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11GLISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11GLISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s2, s2, 3
-; GFX11GLISEL-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11GLISEL-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11GLISEL-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11GLISEL-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX11GLISEL-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11GLISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11GLISEL-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11GLISEL-FAKE16-NEXT: s_endpgm
%result = call i1 @llvm.is.fpclass.f16(half %x, i32 3)
%sext = sext i1 %result to i32
@@ -202,17 +258,28 @@ define i1 @allflags_f16(half %x) nounwind {
}
define i1 @snan_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: snan_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7e00
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e64 s[4:5], s4, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: snan_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7e00
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e64 s[4:5], s4, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: snan_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xffff83ff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x1ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: snan_f16:
; GFX8CHECK: ; %bb.0:
@@ -269,14 +336,24 @@ define i1 @snan_f16(half %x) nounwind {
}
define i1 @qnan_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: qnan_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7dff
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: qnan_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7dff
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: qnan_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7e00
+; GFX7GLISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: qnan_f16:
; GFX8CHECK: ; %bb.0:
@@ -333,14 +410,23 @@ define i1 @qnan_f16(half %x) nounwind {
}
define i1 @posinf_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: posinf_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: posinf_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: posinf_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: posinf_f16:
; GFX8CHECK: ; %bb.0:
@@ -399,14 +485,23 @@ define i1 @posinf_f16(half %x) nounwind {
}
define i1 @neginf_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: neginf_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_mov_b32 s4, 0xfc00
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: neginf_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_mov_b32 s4, 0xfc00
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: neginf_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0xfc00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: neginf_f16:
; GFX8CHECK: ; %bb.0:
@@ -463,19 +558,34 @@ define i1 @neginf_f16(half %x) nounwind {
}
define i1 @posnormal_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: posnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x7800
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e64 s[4:5], -1, v1
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: posnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x7800
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e64 s[4:5], -1, v1
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: posnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v0, v2
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: posnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -534,19 +644,34 @@ define i1 @posnormal_f16(half %x) nounwind {
}
define i1 @negnormal_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: negnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x7800
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: negnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x7800
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: negnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], v0, v2
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: negnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -603,15 +728,25 @@ define i1 @negnormal_f16(half %x) nounwind {
}
define i1 @possubnormal_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: possubnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, -1, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x3ff
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: possubnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, -1, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x3ff
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: possubnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: possubnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -670,18 +805,33 @@ define i1 @possubnormal_f16(half %x) nounwind {
}
define i1 @negsubnormal_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: negsubnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e64 v0, s[4:5], -1, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x3ff
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: negsubnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e64 v0, s[4:5], -1, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x3ff
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: negsubnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cmp_ne_u32_e32 vcc, v0, v2
+; GFX7GLISEL-NEXT: v_add_i32_e64 v0, s[4:5], -1, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e64 s[4:5], v0, v1
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: negsubnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -801,14 +951,23 @@ define i1 @poszero_f16(half %x) nounwind {
}
define i1 @negzero_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: negzero_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_mov_b32 s4, 0x8000
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: negzero_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_mov_b32 s4, 0x8000
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: negzero_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x8000
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: negzero_f16:
; GFX8CHECK: ; %bb.0:
@@ -865,14 +1024,23 @@ define i1 @negzero_f16(half %x) nounwind {
}
define i1 @posfinite_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: posfinite_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: posfinite_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: posfinite_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: posfinite_f16:
; GFX8CHECK: ; %bb.0:
@@ -931,17 +1099,30 @@ define i1 @posfinite_f16(half %x) nounwind {
}
define i1 @negfinite_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: negfinite_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e64 s[4:5], s4, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: negfinite_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e64 s[4:5], s4, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: negfinite_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v0, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e64 s[4:5], v1, v0
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: negfinite_f16:
; GFX8CHECK: ; %bb.0:
@@ -998,14 +1179,24 @@ define i1 @negfinite_f16(half %x) nounwind {
}
define i1 @isnan_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: isnan_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isnan_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isnan_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isnan_f16:
; GFX8CHECK: ; %bb.0:
@@ -1062,14 +1253,24 @@ define i1 @isnan_f16(half %x) nounwind {
}
define i1 @not_isnan_f16(half %x) {
-; GFX7CHECK-LABEL: not_isnan_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c01
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_isnan_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c01
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_isnan_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c01
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_isnan_f16:
; GFX8CHECK: ; %bb.0:
@@ -1215,20 +1416,39 @@ define <2 x i1> @isnan_v2f16(<2 x half> %x) nounwind {
}
define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
-; GFX7CHECK-LABEL: isnan_v3f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v2, 0x7fff, v1
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_bfe_u32 v1, v0, 16, 15
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v1
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v2
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isnan_v3f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v2, 0x7fff, v1
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_bfe_u32 v1, v0, 16, 15
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v1
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v2
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isnan_v3f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v3, 0x7c00
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v0, v3
+; GFX7GLISEL-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v2, v3
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v4, v3
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isnan_v3f16:
; GFX8CHECK: ; %bb.0:
@@ -1325,23 +1545,47 @@ define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
}
define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
-; GFX7CHECK-LABEL: isnan_v4f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_bfe_u32 v3, v1, 16, 15
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_and_b32_e32 v2, 0x7fff, v1
-; GFX7CHECK-NEXT: v_bfe_u32 v1, v0, 16, 15
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v1
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v2
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v3
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isnan_v4f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_bfe_u32 v3, v1, 16, 15
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_and_b32_e32 v2, 0x7fff, v1
+; GFX7SELDAG-NEXT: v_bfe_u32 v1, v0, 16, 15
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v1
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v2
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v3
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isnan_v4f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v4, 0x7c00
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX7GLISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v0, v4
+; GFX7GLISEL-NEXT: v_and_b32_e32 v5, 0xffff, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v3
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v2, v4
+; GFX7GLISEL-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v5, v4
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v3, v4
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isnan_v4f16:
; GFX8CHECK: ; %bb.0:
@@ -1454,14 +1698,24 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
}
define i1 @isnan_f16_strictfp(half %x) strictfp nounwind {
-; GFX7CHECK-LABEL: isnan_f16_strictfp:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isnan_f16_strictfp:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isnan_f16_strictfp:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isnan_f16_strictfp:
; GFX8CHECK: ; %bb.0:
@@ -1518,14 +1772,24 @@ define i1 @isnan_f16_strictfp(half %x) strictfp nounwind {
}
define i1 @isinf_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: isinf_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isinf_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isinf_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isinf_f16:
; GFX8CHECK: ; %bb.0:
@@ -1584,15 +1848,25 @@ define i1 @isinf_f16(half %x) nounwind {
}
define i1 @isfinite_f16(half %x) nounwind {
-; GFX7CHECK-LABEL: isfinite_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xfffe, v0
-; GFX7CHECK-NEXT: s_mov_b32 s4, 0xf800
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isfinite_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xfffe, v0
+; GFX7SELDAG-NEXT: s_mov_b32 s4, 0xf800
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isfinite_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isfinite_f16:
; GFX8CHECK: ; %bb.0:
@@ -1651,13 +1925,22 @@ define i1 @isfinite_f16(half %x) nounwind {
}
define i1 @issubnormal_or_zero_f16(half %x) {
-; GFX7CHECK-LABEL: issubnormal_or_zero_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7c00, v0
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: issubnormal_or_zero_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7c00, v0
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: issubnormal_or_zero_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7c00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: issubnormal_or_zero_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -1717,13 +2000,28 @@ entry:
}
define i1 @not_issubnormal_or_zero_f16(half %x) {
-; GFX7CHECK-LABEL: not_issubnormal_or_zero_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7c00, v0
-; GFX7CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_issubnormal_or_zero_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7c00, v0
+; GFX7SELDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_issubnormal_or_zero_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_issubnormal_or_zero_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -1783,16 +2081,27 @@ entry:
}
define i1 @isnormal_f16(half %x) {
-; GFX7CHECK-LABEL: isnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7800
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7800
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -1851,16 +2160,32 @@ define i1 @isnormal_f16(half %x) {
}
define i1 @not_isnormal_f16(half %x) {
-; GFX7CHECK-LABEL: not_isnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x77ff
-; GFX7CHECK-NEXT: v_cmp_lt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_isnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x77ff
+; GFX7SELDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_isnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7c00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v0, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v0, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_isnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -1919,19 +2244,43 @@ define i1 @not_isnormal_f16(half %x) {
}
define i1 @not_is_plus_normal_f16(half %x) {
-; GFX7CHECK-LABEL: not_is_plus_normal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x77ff
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
-; GFX7CHECK-NEXT: v_cmp_lt_u32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_is_plus_normal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x77ff
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v1
+; GFX7SELDAG-NEXT: v_cmp_lt_u32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_is_plus_normal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7c00, v0
+; GFX7GLISEL-NEXT: v_cmp_ne_u32_e64 s[4:5], v2, v3
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v3, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v3, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[6:7], s[6:7], vcc
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_is_plus_normal_f16:
; GFX8CHECK: ; %bb.0:
@@ -1990,19 +2339,43 @@ define i1 @not_is_plus_normal_f16(half %x) {
}
define i1 @not_is_neg_normal_f16(half %x) {
-; GFX7CHECK-LABEL: not_is_neg_normal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x77ff
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e64 s[4:5], -1, v1
-; GFX7CHECK-NEXT: v_cmp_lt_u32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_is_neg_normal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x77ff
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e64 s[4:5], -1, v1
+; GFX7SELDAG-NEXT: v_cmp_lt_u32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_is_neg_normal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7c00, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v2, v3
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[6:7], v3, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v3, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[6:7], s[6:7], vcc
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_is_neg_normal_f16:
; GFX8CHECK: ; %bb.0:
@@ -2061,15 +2434,26 @@ define i1 @not_is_neg_normal_f16(half %x) {
}
define i1 @issubnormal_f16(half %x) {
-; GFX7CHECK-LABEL: issubnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, -1, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x3ff
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: issubnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, -1, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x3ff
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: issubnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: issubnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -2128,15 +2512,34 @@ define i1 @issubnormal_f16(half %x) {
}
define i1 @not_issubnormal_f16(half %x) {
-; GFX7CHECK-LABEL: not_issubnormal_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, -1, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x3fe
-; GFX7CHECK-NEXT: v_cmp_lt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_issubnormal_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, -1, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x3fe
+; GFX7SELDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_issubnormal_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_issubnormal_f16:
; GFX8CHECK: ; %bb.0:
@@ -2195,13 +2598,22 @@ define i1 @not_issubnormal_f16(half %x) {
}
define i1 @iszero_f16(half %x) {
-; GFX7CHECK-LABEL: iszero_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: iszero_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: iszero_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: iszero_f16:
; GFX8CHECK: ; %bb.0:
@@ -2260,13 +2672,35 @@ define i1 @iszero_f16(half %x) {
}
define i1 @not_iszero_f16(half %x) {
-; GFX7CHECK-LABEL: not_iszero_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_iszero_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_iszero_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v1, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_iszero_f16:
; GFX8CHECK: ; %bb.0:
@@ -2325,14 +2759,23 @@ define i1 @not_iszero_f16(half %x) {
}
define i1 @ispositive_f16(half %x) {
-; GFX7CHECK-LABEL: ispositive_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c01
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: ispositive_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c01
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: ispositive_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c01
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: ispositive_f16:
; GFX8CHECK: ; %bb.0:
@@ -2391,23 +2834,41 @@ define i1 @ispositive_f16(half %x) {
}
define i1 @not_ispositive_f16(half %x) {
-; GFX7CHECK-LABEL: not_ispositive_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e64 s[4:5], s6, v0
-; GFX7CHECK-NEXT: s_mov_b32 s7, 0xfc00
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s7, v1
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_ispositive_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e64 s[4:5], s6, v0
+; GFX7SELDAG-NEXT: s_mov_b32 s7, 0xfc00
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s7, v1
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_ispositive_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v3, 0xfc00
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_ispositive_f16:
; GFX8CHECK: ; %bb.0:
@@ -2464,21 +2925,37 @@ define i1 @not_ispositive_f16(half %x) {
}
define i1 @isnegative_f16(half %x) {
-; GFX7CHECK-LABEL: isnegative_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e64 s[4:5], s4, v0
-; GFX7CHECK-NEXT: s_mov_b32 s6, 0xfc00
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s6, v1
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isnegative_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e64 s[4:5], s4, v0
+; GFX7SELDAG-NEXT: s_mov_b32 s6, 0xfc00
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v1
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isnegative_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0xfc00
+; GFX7GLISEL-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isnegative_f16:
; GFX8CHECK: ; %bb.0:
@@ -2535,18 +3012,32 @@ define i1 @isnegative_f16(half %x) {
}
define i1 @not_isnegative_f16(half %x) {
-; GFX7CHECK-LABEL: not_isnegative_f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c01
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v1
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_isnegative_f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c01
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v1
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_isnegative_f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c01
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e64 s[4:5], v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_isnegative_f16:
; GFX8CHECK: ; %bb.0:
@@ -2605,16 +3096,27 @@ define i1 @not_isnegative_f16(half %x) {
}
define i1 @iszero_or_nan_f16(half %x) {
-; GFX7CHECK-LABEL: iszero_or_nan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: iszero_or_nan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: iszero_or_nan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xffff83ff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0xffff8400
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: iszero_or_nan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -2674,16 +3176,27 @@ entry:
}
define i1 @iszero_or_nan_f_daz(half %x) #0 {
-; GFX7CHECK-LABEL: iszero_or_nan_f_daz:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: iszero_or_nan_f_daz:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: iszero_or_nan_f_daz:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xffff83ff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0xffff8400
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: iszero_or_nan_f_daz:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -2743,16 +3256,27 @@ entry:
}
define i1 @iszero_or_nan_f_maybe_daz(half %x) #1 {
-; GFX7CHECK-LABEL: iszero_or_nan_f_maybe_daz:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: iszero_or_nan_f_maybe_daz:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: iszero_or_nan_f_maybe_daz:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xffff83ff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0xffff8400
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: iszero_or_nan_f_maybe_daz:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -2812,16 +3336,36 @@ entry:
}
define i1 @not_iszero_or_nan_f16(half %x) {
-; GFX7CHECK-LABEL: not_iszero_or_nan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c01
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_iszero_or_nan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c01
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_iszero_or_nan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v1, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_iszero_or_nan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -2881,16 +3425,36 @@ entry:
}
define i1 @not_iszero_or_nan_f_daz(half %x) #0 {
-; GFX7CHECK-LABEL: not_iszero_or_nan_f_daz:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c01
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_iszero_or_nan_f_daz:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c01
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_iszero_or_nan_f_daz:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v1, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_iszero_or_nan_f_daz:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -2950,16 +3514,36 @@ entry:
}
define i1 @not_iszero_or_nan_f_maybe_daz(half %x) #1 {
-; GFX7CHECK-LABEL: not_iszero_or_nan_f_maybe_daz:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c01
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_iszero_or_nan_f_maybe_daz:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c01
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_iszero_or_nan_f_maybe_daz:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v1, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_iszero_or_nan_f_maybe_daz:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3019,16 +3603,27 @@ entry:
}
define i1 @iszero_or_qnan_f16(half %x) {
-; GFX7CHECK-LABEL: iszero_or_qnan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7dff
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: iszero_or_qnan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7dff
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: iszero_or_qnan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xffff8200, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0xffff8201
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: iszero_or_qnan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3088,19 +3683,32 @@ entry:
}
define i1 @iszero_or_snan_f16(half %x) {
-; GFX7CHECK-LABEL: iszero_or_snan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7e00
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e64 s[4:5], s4, v0
-; GFX7CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: iszero_or_snan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7e00
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e64 s[4:5], s4, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: iszero_or_snan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xffff83ff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x1ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: iszero_or_snan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3160,28 +3768,52 @@ entry:
}
define i1 @not_iszero_or_qnan_f16(half %x) {
-; GFX7CHECK-LABEL: not_iszero_or_qnan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7e00
-; GFX7CHECK-NEXT: s_movk_i32 s8, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e64 s[4:5], s8, v0
-; GFX7CHECK-NEXT: s_and_b64 s[6:7], s[4:5], vcc
-; GFX7CHECK-NEXT: v_add_i32_e64 v1, s[4:5], -1, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x3ff
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s8, v0
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v1
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x7800
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_iszero_or_qnan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7e00
+; GFX7SELDAG-NEXT: s_movk_i32 s8, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e64 s[4:5], s8, v0
+; GFX7SELDAG-NEXT: s_and_b64 s[6:7], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_add_i32_e64 v1, s[4:5], -1, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x3ff
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s8, v0
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v1
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x7800
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_iszero_or_qnan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v1, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_add_i32_e32 v1, vcc, 0xffff83ff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x1ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_iszero_or_qnan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3241,26 +3873,49 @@ entry:
}
define i1 @not_iszero_or_snan_f16(half %x) {
-; GFX7CHECK-LABEL: not_iszero_or_snan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_add_i32_e64 v1, s[4:5], -1, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x3ff
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v1
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x7dff
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s6, 0x7800
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
-; GFX7CHECK-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_iszero_or_snan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_add_i32_e64 v1, s[4:5], -1, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x3ff
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], s4, v1
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x7dff
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s6, 0x7800
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s6, v0
+; GFX7SELDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_iszero_or_snan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_add_i32_e32 v1, vcc, -1, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x3ff
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v2
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7GLISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_add_i32_e32 v0, vcc, 0xfffffc00, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7800
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_iszero_or_snan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3320,15 +3975,25 @@ entry:
}
define i1 @isinf_or_nan_f16(half %x) {
-; GFX7CHECK-LABEL: isinf_or_nan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xfffe, v0
-; GFX7CHECK-NEXT: s_mov_b32 s4, 0xf7ff
-; GFX7CHECK-NEXT: v_cmp_lt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isinf_or_nan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xfffe, v0
+; GFX7SELDAG-NEXT: s_mov_b32 s4, 0xf7ff
+; GFX7SELDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isinf_or_nan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isinf_or_nan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3388,15 +4053,25 @@ entry:
}
define i1 @not_isinf_or_nan_f16(half %x) {
-; GFX7CHECK-LABEL: not_isinf_or_nan_f16:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xfffe, v0
-; GFX7CHECK-NEXT: s_mov_b32 s4, 0xf800
-; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_isinf_or_nan_f16:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xfffe, v0
+; GFX7SELDAG-NEXT: s_mov_b32 s4, 0xf800
+; GFX7SELDAG-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_isinf_or_nan_f16:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_isinf_or_nan_f16:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3456,14 +4131,24 @@ entry:
}
define i1 @isfinite_or_nan_f(half %x) {
-; GFX7CHECK-LABEL: isfinite_or_nan_f:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isfinite_or_nan_f:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_ne_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: isfinite_or_nan_f:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isfinite_or_nan_f:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3523,14 +4208,24 @@ entry:
}
define i1 @not_isfinite_or_nan_f(half %x) {
-; GFX7CHECK-LABEL: not_isfinite_or_nan_f:
-; GFX7CHECK: ; %bb.0: ; %entry
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: not_isfinite_or_nan_f:
+; GFX7SELDAG: ; %bb.0: ; %entry
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7GLISEL-LABEL: not_isfinite_or_nan_f:
+; GFX7GLISEL: ; %bb.0: ; %entry
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v1, 0x7c00
+; GFX7GLISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: not_isfinite_or_nan_f:
; GFX8CHECK: ; %bb.0: ; %entry
@@ -3600,13 +4295,5 @@ attributes #0 = { denormal_fpenv(ieee|preservesign) }
; Maybe daz
attributes #1 = { denormal_fpenv(ieee|dynamic) }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10GLISEL: {{.*}}
-; GFX10SELDAG: {{.*}}
; GFX11GLISEL: {{.*}}
; GFX11SELDAG: {{.*}}
-; GFX7GLISEL: {{.*}}
-; GFX7SELDAG: {{.*}}
-; GFX8GLISEL: {{.*}}
-; GFX8SELDAG: {{.*}}
-; GFX9GLISEL: {{.*}}
-; GFX9SELDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
index baefe561e94ae..ab6050b4b0b67 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
@@ -8,7 +8,7 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL-TRUE16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL-FAKE16 %s
define float @test_ldexp_f32_i32(ptr addrspace(1) %out, float %a, i32 %b) {
@@ -201,17 +201,17 @@ define half @test_ldexp_f16_i8(half %a, i8 %b) {
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_ldexp_f16_i8:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_ldexp_f16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_ldexp_f16_i8:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_f16_i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ldexp_f16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_f16_i8:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_f16_i8:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -229,6 +229,26 @@ define half @test_ldexp_f16_i8(half %a, i8 %b) {
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX8-GISEL-LABEL: test_ldexp_f16_i8:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v1, v1, v2, v3
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_ldexp_f16_i8:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v1, v1, v2, v3
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_f16_i8:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -243,7 +263,9 @@ define half @test_ldexp_f16_i8(half %a, i8 %b) {
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, 0xffff8000, v1, v2
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.ldexp.f16.i8(half %a, i8 %b)
@@ -308,23 +330,23 @@ define half @test_ldexp_f16_i32(half %a, i32 %b) {
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_ldexp_f16_i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX8-NEXT: v_med3_i32 v1, v1, s4, v2
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_ldexp_f16_i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_ldexp_f16_i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v1, v1, s4, v2
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_ldexp_f16_i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v1, v1, s4, v2
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_f16_i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -344,6 +366,24 @@ define half @test_ldexp_f16_i32(half %a, i32 %b) {
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX8-GISEL-LABEL: test_ldexp_f16_i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v1, v1, v2, v3
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_ldexp_f16_i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v1, v1, v2, v3
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_f16_i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -356,9 +396,9 @@ define half @test_ldexp_f16_i32(half %a, i32 %b) {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_f16_i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v1, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, 0xffff8000, v1, v2
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.ldexp.f16.i32(half %a, i32 %b)
@@ -1091,7 +1131,3 @@ attributes #0 = { nounwind readnone }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX6-GISEL: {{.*}}
; GFX6-SDAG: {{.*}}
-; GFX8-GISEL: {{.*}}
-; GFX8-SDAG: {{.*}}
-; GFX9-GISEL: {{.*}}
-; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index 473711b77c126..d131bc2955457 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -3714,21 +3714,37 @@ define float @v_log_f32_0() {
}
define float @v_log_f32_from_fpext_f16(i16 %src.i) {
-; SI-LABEL: v_log_f32_from_fpext_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x3f317217
-; SI-NEXT: s_mov_b32 s5, 0x3377d1cf
-; SI-NEXT: s_mov_b32 s6, 0x7f800000
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_f32_from_fpext_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3f317217
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3377d1cf
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_log_f32_from_fpext_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_log_f32_from_fpext_f16:
; VI: ; %bb.0:
@@ -3749,21 +3765,37 @@ define float @v_log_f32_from_fpext_f16(i16 %src.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_f32_from_fpext_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x3f317217
-; GFX900-NEXT: s_mov_b32 s5, 0x3377d1cf
-; GFX900-NEXT: s_mov_b32 s6, 0x7f800000
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_f32_from_fpext_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3f317217
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x3377d1cf
+; GFX900-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_f32_from_fpext_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_f32_from_fpext_f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3827,7 +3859,7 @@ define float @v_log_f32_from_fpext_f16(i16 %src.i) {
; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_fmac_f32_e32 v2, 0x3377d1cf, v0
; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -3849,25 +3881,45 @@ define float @v_log_f32_from_fpext_f16(i16 %src.i) {
}
define float @v_log_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
-; SI-LABEL: v_log_f32_from_fpext_math_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x3f317217
-; SI-NEXT: s_mov_b32 s5, 0x3377d1cf
-; SI-NEXT: s_mov_b32 s6, 0x7f800000
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_f32_from_fpext_math_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3f317217
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3377d1cf
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_log_f32_from_fpext_math_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_log_f32_from_fpext_math_f16:
; VI: ; %bb.0:
@@ -3889,22 +3941,39 @@ define float @v_log_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_f32_from_fpext_math_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x3f317217
-; GFX900-NEXT: s_mov_b32 s5, 0x3377d1cf
-; GFX900-NEXT: s_mov_b32 s6, 0x7f800000
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_f32_from_fpext_math_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3f317217
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x3377d1cf
+; GFX900-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_f32_from_fpext_math_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_f32_from_fpext_math_f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3972,7 +4041,7 @@ define float @v_log_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_fmac_f32_e32 v2, 0x3377d1cf, v0
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -4152,11 +4221,11 @@ define half @v_log_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_f16:
@@ -4219,11 +4288,11 @@ define half @v_log_fabs_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_fabs_f16:
@@ -4287,11 +4356,11 @@ define half @v_log_fneg_fabs_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_fneg_fabs_f16:
@@ -4356,11 +4425,11 @@ define half @v_log_fneg_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_fneg_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index 6cbc4d45bfc67..a2e60c075b352 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -3714,21 +3714,37 @@ define float @v_log10_f32_0() {
}
define float @v_log10_f32_from_fpext_f16(i16 %src.i) {
-; SI-LABEL: v_log10_f32_from_fpext_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x3e9a209a
-; SI-NEXT: s_mov_b32 s5, 0x3284fbcf
-; SI-NEXT: s_mov_b32 s6, 0x7f800000
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_f32_from_fpext_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3e9a209a
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3284fbcf
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_log10_f32_from_fpext_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_log10_f32_from_fpext_f16:
; VI: ; %bb.0:
@@ -3749,21 +3765,37 @@ define float @v_log10_f32_from_fpext_f16(i16 %src.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_f32_from_fpext_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x3e9a209a
-; GFX900-NEXT: s_mov_b32 s5, 0x3284fbcf
-; GFX900-NEXT: s_mov_b32 s6, 0x7f800000
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_f32_from_fpext_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3e9a209a
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x3284fbcf
+; GFX900-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_f32_from_fpext_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_f32_from_fpext_f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3827,7 +3859,7 @@ define float @v_log10_f32_from_fpext_f16(i16 %src.i) {
; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_fmac_f32_e32 v2, 0x3284fbcf, v0
; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -3849,25 +3881,45 @@ define float @v_log10_f32_from_fpext_f16(i16 %src.i) {
}
define float @v_log10_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
-; SI-LABEL: v_log10_f32_from_fpext_math_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: s_mov_b32 s4, 0x3e9a209a
-; SI-NEXT: s_mov_b32 s5, 0x3284fbcf
-; SI-NEXT: s_mov_b32 s6, 0x7f800000
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; SI-NEXT: v_fma_f32 v2, v0, s4, -v1
-; SI-NEXT: v_fma_f32 v2, v0, s5, v2
-; SI-NEXT: v_add_f32_e32 v1, v1, v2
-; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_f32_from_fpext_math_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, 0x3e9a209a
+; SI-SDAG-NEXT: s_mov_b32 s5, 0x3284fbcf
+; SI-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; SI-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; SI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_log10_f32_from_fpext_math_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_log10_f32_from_fpext_math_f16:
; VI: ; %bb.0:
@@ -3889,22 +3941,39 @@ define float @v_log10_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_f32_from_fpext_math_f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: s_mov_b32 s4, 0x3e9a209a
-; GFX900-NEXT: s_mov_b32 s5, 0x3284fbcf
-; GFX900-NEXT: s_mov_b32 s6, 0x7f800000
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX900-NEXT: v_fma_f32 v2, v0, s4, -v1
-; GFX900-NEXT: v_fma_f32 v2, v0, s5, v2
-; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX900-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_f32_from_fpext_math_f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3e9a209a
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x3284fbcf
+; GFX900-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s4, -v1
+; GFX900-SDAG-NEXT: v_fma_f32 v2, v0, s5, v2
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s6
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_f32_from_fpext_math_f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_f32_from_fpext_math_f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3972,7 +4041,7 @@ define float @v_log10_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_fmac_f32_e32 v2, 0x3284fbcf, v0
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -4152,11 +4221,11 @@ define half @v_log10_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_f16:
@@ -4219,11 +4288,11 @@ define half @v_log10_fabs_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_fabs_f16:
@@ -4287,11 +4356,11 @@ define half @v_log10_fneg_fabs_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_fneg_fabs_f16:
@@ -4356,11 +4425,11 @@ define half @v_log10_fneg_f16(half %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, 0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_fneg_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index 3f2e2edc1f26c..c059eae6cc153 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -2131,16 +2131,27 @@ define float @v_log2_f32_from_fpext_f16(i16 %src.i) {
}
define float @v_log2_f32_from_fpext_math_f16(i16 %src0.i, i16 %src1.i) {
-; SI-LABEL: v_log2_f32_from_fpext_math_f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v0, v0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_f32_from_fpext_math_f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_log2_f32_from_fpext_math_f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_log2_f32_from_fpext_math_f16:
; VI: ; %bb.0:
@@ -3403,7 +3414,5 @@ attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memo
; GFX689-SDAG: {{.*}}
; GFX900-GISEL: {{.*}}
; GFX900-SDAG: {{.*}}
-; SI-GISEL: {{.*}}
-; SI-SDAG: {{.*}}
; VI-GISEL: {{.*}}
; VI-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
index 76aef501dd79e..bc980135e79fd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
@@ -12,33 +12,59 @@ declare { double, double } @llvm.modf.f64(double)
declare { <2 x double>, <2 x double> } @llvm.modf.v2f64(<2 x double>)
define { half, half } @test_modf_f16(half %x) {
-; GFX9-LABEL: test_modf_f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x7c00
-; GFX9-NEXT: v_trunc_f16_e32 v1, v0
-; GFX9-NEXT: v_sub_f16_e32 v2, v0, v1
-; GFX9-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_bfi_b32 v0, s4, v2, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX9-SDAG-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-SDAG-NEXT: v_sub_f16_e32 v2, v0, v1
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_bfi_b32 v0, s4, v2, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7c00
+; GFX9-GISEL-NEXT: v_sub_f16_e32 v2, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[4:5], |v0|, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff8000, v0
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { half, half } @llvm.modf.f16(half %x)
ret { half, half } %result
}
define half @test_modf_f16_only_use_fract(half %x) {
-; GFX9-LABEL: test_modf_f16_only_use_fract:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x7c00
-; GFX9-NEXT: v_trunc_f16_e32 v1, v0
-; GFX9-NEXT: v_sub_f16_e32 v1, v0, v1
-; GFX9-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_bfi_b32 v0, s4, v1, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_f16_only_use_fract:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX9-SDAG-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-SDAG-NEXT: v_sub_f16_e32 v1, v0, v1
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_bfi_b32 v0, s4, v1, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_f16_only_use_fract:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX9-GISEL-NEXT: v_sub_f16_e32 v1, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[4:5], |v0|, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff8000, v0
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { half, half } @llvm.modf.f16(half %x)
%result.0 = extractvalue { half, half } %result, 0
ret half %result.0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
index 78c8ed13cab1c..887673cc23f10 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GFX678,GFX6,GFX678-SDAG,GFX6-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GFX678,GFX6,GFX678-GISEL,GFX6-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GFX678,GFX6,GFX678-GISEL,GFX6-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GFX678,GFX7,GFX678-SDAG,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GFX678,GFX7,GFX678-GISEL,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GFX678,GFX7,GFX678-GISEL,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX678,GFX678-SDAG,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX678,GFX678-GISEL,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=GFX678,GFX678-GISEL,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX1011,GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX1011,GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GFX1011,GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1011,GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1011,GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1011,GFX11,GFX11-GISEL %s
declare void @llvm.set.rounding(i32)
declare i32 @llvm.get.rounding()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index 83eae6f348d79..4e9e00c96aa64 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -2,11 +2,11 @@
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=SI %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=VI %s
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-TRUE16,GFX11-TRUE16-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-TRUE16,GFX11-TRUE16-GISEL %s
+; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-TRUE16,GFX11-TRUE16-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-FAKE16,GFX11-FAKE16-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-FAKE16,GFX11-FAKE16-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-TRUE16,GFX12-TRUE16-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-TRUE16,GFX12-TRUE16-GISEL %s
+; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-TRUE16,GFX12-TRUE16-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-FAKE16,GFX12-FAKE16-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-FAKE16,GFX12-FAKE16-GISEL %s
diff --git a/llvm/test/CodeGen/AMDGPU/lround.ll b/llvm/test/CodeGen/AMDGPU/lround.ll
index f6f021057d60f..4f7bdc52e4ebf 100644
--- a/llvm/test/CodeGen/AMDGPU/lround.ll
+++ b/llvm/test/CodeGen/AMDGPU/lround.ll
@@ -805,8 +805,9 @@ define half @intrinsic_fround_half(half %arg) {
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x3c00
; GFX9-GISEL-NEXT: v_cmp_ge_f16_e64 vcc, |v2|, 0.5
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v3, vcc
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-GISEL-NEXT: v_bfi_b32 v0, s4, v2, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff8000, v0
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_add_f16_e32 v0, v1, v0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -826,9 +827,11 @@ define half @intrinsic_fround_half(half %arg) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_trunc_f16_e32 v1, v0
; GFX10-GISEL-NEXT: v_sub_f16_e32 v2, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff8000, v0
; GFX10-GISEL-NEXT: v_cmp_ge_f16_e64 s4, |v2|, 0.5
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 0x3c00, s4
-; GFX10-GISEL-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_add_f16_e32 v0, v1, v0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -881,13 +884,15 @@ define half @intrinsic_fround_half(half %arg) {
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_trunc_f16_e32 v1, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-FAKE16-NEXT: v_sub_f16_e32 v2, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff8000, v0
; GFX11-GISEL-FAKE16-NEXT: v_cmp_ge_f16_e64 s0, |v2|, 0.5
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x3c00, s0
-; GFX11-GISEL-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0
; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v1, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index 1c24a2c8e3798..8041d30871d14 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -7,8 +7,8 @@
; FIXME-TRUE16. fix gisel
; XUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GISEL-GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GISEL-GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI,GISEL-VI %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GISEL-CI %s
@@ -25,15 +25,15 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo(half %src0, half %s
; GFX9-NEXT: v_mad_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo:
; SDAG-CI: ; %bb.0:
@@ -46,6 +46,18 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo(half %src0, half %s
; SDAG-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -221,30 +233,30 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
}
define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, half %src2) #0 {
-; GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
; SDAG-CI: ; %bb.0:
@@ -257,6 +269,35 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, ha
; SDAG-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -265,6 +306,7 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, ha
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext half %src0 to float
@@ -279,30 +321,30 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, ha
}
define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext(half %src0, half %src1, half %src2) #0 {
-; GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
; SDAG-CI: ; %bb.0:
@@ -315,6 +357,35 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext(half %src0, half %src
; SDAG-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GISEL-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-GFX9-NEXT: v_lshlrev_b32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -323,6 +394,7 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext(half %src0, half %src
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-CI-NEXT: v_bfe_i32 v0, v0, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext half %src0 to float
@@ -354,22 +426,22 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt(half %
; SDAG-GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; GFX9-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX9-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
; SDAG-CI: ; %bb.0:
@@ -386,11 +458,34 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt(half %
; GISEL-GFX11: ; %bb.0:
; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_and_b32_e64 v1, 0xffff, s0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GISEL-GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GISEL-GFX11-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; GISEL-GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
+; GISEL-GFX9-NEXT: v_and_b32_e32 v1, s4, v1
+; GISEL-GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -426,15 +521,15 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt(half
; GFX9-NEXT: v_mad_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt:
; SDAG-CI: ; %bb.0:
@@ -453,6 +548,18 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt(half
; SDAG-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v0, v2 clamp
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -511,18 +618,18 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi
; GFX9-NEXT: v_mad_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v2
-; VI-NEXT: flat_store_short v[0:1], v0
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_f16_sdwa v0, v0, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; SDAG-VI-NEXT: flat_store_short v[0:1], v0
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0)
+; SDAG-VI-NEXT: v_max_f16_sdwa v0, v0, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
; SDAG-CI: ; %bb.0:
@@ -554,6 +661,21 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi
; GISEL-GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: flat_store_short v[0:1], v0
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0)
+; GISEL-VI-NEXT: v_max_f16_e64 v0, v0, v0 clamp
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -598,8 +720,3 @@ attributes #0 = { nounwind denormal_fpenv(float: preservesign) }
attributes #1 = { nounwind readnone speculatable }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GISEL-GFX11-FAKE16: {{.*}}
-; GISEL-GFX9: {{.*}}
-; GISEL-VI: {{.*}}
-; SDAG-GFX11: {{.*}}
-; SDAG-GFX9: {{.*}}
-; SDAG-VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index d5514894cb2dc..3596c8d9aaa7e 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -2239,12 +2239,12 @@ define i32 @mixlo_zext(float %src0, float %src1, float %src2) #0 {
; GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: mixlo_zext:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: mixlo_zext:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: mixlo_zext:
; SDAG-CI: ; %bb.0:
@@ -2261,11 +2261,20 @@ define i32 @mixlo_zext(float %src0, float %src1, float %src2) #0 {
; GISEL-GFX1100-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: mixlo_zext:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: mixlo_zext:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2)
%cvt.result = fptrunc float %result to half
@@ -2488,5 +2497,3 @@ attributes #0 = { nounwind denormal_fpenv(float: preservesign) }
attributes #1 = { nounwind readnone speculatable }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GISEL-GFX1100-FAKE16: {{.*}}
-; GISEL-VI: {{.*}}
-; SDAG-VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix.ll b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
index 4e1acf114da5d..449b56e20dd3d 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
@@ -359,23 +359,23 @@ define float @v_mad_mix_f32_negf16lo_f16lo_f16lo(half %src0, half %src1, half %s
; GFX906-NEXT: v_fma_mix_f32 v0, -v0, v1, v2 op_sel_hi:[1,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mad_f32 v0, -v0, v1, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
; SDAG-CI: ; %bb.0:
@@ -386,6 +386,24 @@ define float @v_mad_mix_f32_negf16lo_f16lo_f16lo(half %src0, half %src1, half %s
; SDAG-CI-NEXT: v_mad_f32 v0, -v0, v1, v2
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -729,27 +747,27 @@ define float @v_mad_mix_f32_f16lo_f16lo_negabsf32(half %src0, half %src1, float
; inline immediate.
define float @v_mad_mix_f32_f16lo_f16lo_f32imm1(half %src0, half %src1) #0 {
-; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: s_mov_b32 s0, 1.0
-; GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: s_mov_b32 s0, 1.0
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_mov_b32 s4, 1.0
-; GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: s_mov_b32 s4, 1.0
+; SDAG-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: s_mov_b32 s4, 1.0
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: s_mov_b32 s4, 1.0
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
; GFX9GEN: ; %bb.0:
@@ -774,6 +792,28 @@ define float @v_mad_mix_f32_f16lo_f16lo_f32imm1(half %src0, half %src1) #0 {
; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; CI-NEXT: v_mad_f32 v0, v0, v1, 1.0
; CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_mov_b32_e32 v2, 1.0
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_mov_b32_e32 v2, 1.0
+; GISEL-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_mov_b32_e32 v2, 1.0
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float 1.0)
@@ -781,27 +821,27 @@ define float @v_mad_mix_f32_f16lo_f16lo_f32imm1(half %src0, half %src1) #0 {
}
define float @v_mad_mix_f32_f16lo_f16lo_f32imminv2pi(half %src0, half %src1) #0 {
-; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: s_mov_b32 s0, 0.15915494
-; GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: s_mov_b32 s0, 0.15915494
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_mov_b32 s4, 0.15915494
-; GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: s_mov_b32 s4, 0.15915494
+; SDAG-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: s_mov_b32 s4, 0.15915494
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: s_mov_b32 s4, 0.15915494
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
; GFX9GEN: ; %bb.0:
@@ -827,6 +867,28 @@ define float @v_mad_mix_f32_f16lo_f16lo_f32imminv2pi(half %src0, half %src1) #0
; SDAG-CI-NEXT: v_madak_f32 v0, v0, v1, 0x3e22f983
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_mov_b32_e32 v2, 0.15915494
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_mov_b32_e32 v2, 0.15915494
+; GISEL-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_mov_b32_e32 v2, 0.15915494
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -848,43 +910,43 @@ define float @v_mad_mix_f32_f16lo_f16lo_f32imminv2pi(half %src0, half %src1) #0
; f32 1/2pi = 0x3e22f983
define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi(half %src0, half %src1) #0 {
-; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: s_mov_b32 s0, 0x3e230000
-; GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_mov_b32 s4, 0x3e230000
-; GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: s_mov_b32 s0, 0x3e230000
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: s_mov_b32 s4, 0x3e230000
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: s_mov_b32 s4, 0x3e230000
+; SDAG-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_madak_f32 v0, v0, v1, 0x3e230000
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: s_mov_b32 s4, 0x3e230000
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_madak_f32 v0, v0, v1, 0x3e230000
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_madak_f32 v0, v0, v1, 0x3e230000
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_madak_f32 v0, v0, v1, 0x3e230000
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
; SDAG-CI: ; %bb.0:
@@ -894,6 +956,46 @@ define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi(half %src0, half %src1)
; SDAG-CI-NEXT: v_madak_f32 v0, v0, v1, 0x3e230000
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_mov_b32_e32 v2, 0x3e230000
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_mov_b32_e32 v2, 0x3e230000
+; GISEL-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_mov_b32_e32 v2, 0x3e230000
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_mov_b32_e32 v0, 0x3e230000
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v2, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_mov_b32_e32 v0, 0x3e230000
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v2, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -911,43 +1013,43 @@ define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi(half %src0, half %src1)
define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imm63(half %src0, half %src1) #0 {
-; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: s_mov_b32 s0, 0x367c0000
-; GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_mov_b32 s4, 0x367c0000
-; GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: s_mov_b32 s0, 0x367c0000
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, s0 op_sel_hi:[1,1,0]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: s_mov_b32 s4, 0x367c0000
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: s_mov_b32 s4, 0x367c0000
+; SDAG-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_madak_f32 v0, v0, v1, 0x367c0000
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: s_mov_b32 s4, 0x367c0000
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, s4 op_sel_hi:[1,1,0]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_madak_f32 v0, v0, v1, 0x367c0000
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_madak_f32 v0, v0, v1, 0x367c0000
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_madak_f32 v0, v0, v1, 0x367c0000
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
; SDAG-CI: ; %bb.0:
@@ -957,6 +1059,46 @@ define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imm63(half %src0, half %src1) #0 {
; SDAG-CI-NEXT: v_madak_f32 v0, v0, v1, 0x367c0000
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_mov_b32_e32 v2, 0x367c0000
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_mov_b32_e32 v2, 0x367c0000
+; GISEL-GFX900-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_mov_b32_e32 v2, 0x367c0000
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_mov_b32_e32 v0, 0x367c0000
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v2, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_mov_b32_e32 v0, 0x367c0000
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v2, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1763,23 +1905,23 @@ define float @v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo(i32 %src0.arg, half %src1
; GFX906-NEXT: v_fma_mix_f32 v0, -v0, v1, v2 op_sel_hi:[1,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mad_f32 v0, -v0, v1, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
; SDAG-CI: ; %bb.0:
@@ -1790,6 +1932,24 @@ define float @v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo(i32 %src0.arg, half %src1
; SDAG-CI-NEXT: v_mad_f32 v0, -v0, v1, v2
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2421,41 +2581,41 @@ define <2 x float> @v_mad_mix_v2f32_shuffle_cvt_add(<2 x half> %src0, <2 x half>
}
define float @v_mad_mix_f32_negf16lo_add_f16lo(half %src0, half %src1) {
-; GFX1100-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX900-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_sub_f32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
; SDAG-CI: ; %bb.0:
@@ -2465,6 +2625,42 @@ define float @v_mad_mix_f32_negf16lo_add_f16lo(half %src0, half %src1) {
; SDAG-CI-NEXT: v_sub_f32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_negf16lo_add_f16lo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2920,41 +3116,41 @@ define float @v_mad_mix_clamp_f32_f16hi_add_f16hi(<2 x half> %src0, <2 x half> %
}
define float @v_mad_mix_f32_negprecvtf16lo_add_f16lo(i32 %src0.arg, half %src1) {
-; GFX1100-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX900-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_sub_f32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
; SDAG-CI: ; %bb.0:
@@ -2964,6 +3160,42 @@ define float @v_mad_mix_f32_negprecvtf16lo_add_f16lo(i32 %src0.arg, half %src1)
; SDAG-CI-NEXT: v_sub_f32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_negprecvtf16lo_add_f16lo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3034,41 +3266,41 @@ define float @v_mad_mix_f32_absprecvtf16lo_add_f16lo(i32 %src0.arg, half %src1)
}
define float @v_mad_mix_f32_negabsprecvtf16lo_add_f16lo(i32 %src0.arg, half %src1) {
-; GFX1100-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel_hi:[1,1,1]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX900-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel_hi:[1,1,1]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_sub_f32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
; SDAG-CI: ; %bb.0:
@@ -3078,6 +3310,42 @@ define float @v_mad_mix_f32_negabsprecvtf16lo_add_f16lo(i32 %src0.arg, half %src
; SDAG-CI-NEXT: v_sub_f32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, -|v0|, 1.0, v1 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, -|v0|, 1.0, v1 op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_negabsprecvtf16lo_add_f16lo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3228,11 +3496,11 @@ define float @v_mad_mix_f32_precvtabsf16hi_add_f16lo(i32 %src0.arg, half %src1)
}
define float @v_mad_mix_f32_preextractfneg_f16hi_add_f16lo(i32 %src0.arg, half %src1) {
-; GFX1100-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
; GFX900: ; %bb.0:
@@ -3242,11 +3510,11 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_add_f16lo(i32 %src0.arg, half %
; GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
; GFX9GEN: ; %bb.0:
@@ -3272,6 +3540,18 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_add_f16lo(i32 %src0.arg, half %
; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; CI-NEXT: v_add_f32_e32 v0, v0, v1
; CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fneg = fneg <2 x half> %src0.arg.bc
%src0 = extractelement <2 x half> %fneg, i32 1
@@ -3336,11 +3616,11 @@ define float @v_mad_mix_f32_preextractfabs_f16hi_add_f16lo(i32 %src0.arg, half %
}
define float @v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo(i32 %src0.arg, half %src1) {
-; GFX1100-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
; GFX900: ; %bb.0:
@@ -3350,11 +3630,11 @@ define float @v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo(i32 %src0.arg, ha
; GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
; GFX9GEN: ; %bb.0:
@@ -3380,6 +3660,18 @@ define float @v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo(i32 %src0.arg, ha
; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; CI-NEXT: v_add_f32_e32 v0, v0, v1
; CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX1100-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; GISEL-GFX1100: ; %bb.0:
+; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, -|v0|, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, -|v0|, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
%fneg.fabs = fneg <2 x half> %fabs
@@ -3699,13 +3991,13 @@ define float @v_mad_mix_f32_negf16lo_mul_f16lo(half %src0, half %src1) {
; GFX1100-NEXT: v_fma_mix_f32 v0, -v0, v1, neg(0) op_sel_hi:[1,1,0]
; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
; GFX906: ; %bb.0:
@@ -3713,21 +4005,21 @@ define float @v_mad_mix_f32_negf16lo_mul_f16lo(half %src0, half %src1) {
; GFX906-NEXT: v_fma_mix_f32 v0, -v0, v1, neg(0) op_sel_hi:[1,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; VI-NEXT: v_mul_f32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-VI-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
; SDAG-CI: ; %bb.0:
@@ -3737,6 +4029,30 @@ define float @v_mad_mix_f32_negf16lo_mul_f16lo(half %src0, half %src1) {
; SDAG-CI-NEXT: v_mul_f32_e32 v0, v0, v1
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_f32_negf16lo_mul_f16lo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4851,15 +5167,25 @@ define float @v_mad_mix_f32_absf16hi_fsub_f16hi(i32 %src0, i32 %src1) {
; VI-NEXT: v_sub_f32_e64 v0, |v0|, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_absf16hi_fsub_f16hi:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_sub_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_absf16hi_fsub_f16hi:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_sub_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_absf16hi_fsub_f16hi:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_sub_f32_e64 v0, |v0|, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.hi = lshr i32 %src0, 16
%src1.hi = lshr i32 %src1, 16
%src0.i16 = trunc i32 %src0.hi to i16
@@ -4886,7 +5212,3 @@ attributes #1 = { nounwind denormal_fpenv(float: ieee|ieee) }
attributes #2 = { nounwind readnone speculatable }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GISEL-GFX1100-FAKE16: {{.*}}
-; GISEL-GFX9GEN: {{.*}}
-; GISEL-VI: {{.*}}
-; SDAG-GFX9GEN: {{.*}}
-; SDAG-VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
index dfd0f4533b410..0dd6a1cd06c77 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-SDAG %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-GISEL %s
define void @memset_pattern_i128_len0_dynvalue(ptr addrspace(1) align 16 %a, i128 %value) {
; GFX942-LABEL: memset_pattern_i128_len0_dynvalue:
@@ -317,185 +317,364 @@ define void @memset_pattern_i128_len16(ptr addrspace(1) align 16 %a) {
}
define void @memset_pattern_i128_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
-; GFX942-LABEL: memset_pattern_i128_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB7_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v4, 0xdddddddd
-; GFX942-NEXT: v_mov_b32_e32 v5, 0xcccccccc
-; GFX942-NEXT: v_mov_b32_e32 v6, 0xbbbbbbbb
-; GFX942-NEXT: v_mov_b32_e32 v7, 0xaaaaaaaa
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB7_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX942-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB7_2
-; GFX942-NEXT: .LBB7_3: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i128_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB7_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0xdddddddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0xcccccccc
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, 0xbbbbbbbb
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0xaaaaaaaa
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB7_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB7_2
+; GFX942-SDAG-NEXT: .LBB7_3: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i128_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB7_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s10, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s11, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: .LBB7_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 16, v0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB7_2
+; GFX942-GISEL-NEXT: .LBB7_3: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i128_dynlen_lds(ptr addrspace(3) align 16 %a, i64 %len) {
-; GFX942-LABEL: memset_pattern_i128_dynlen_lds:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v1
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB8_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v2, 0xdddddddd
-; GFX942-NEXT: v_mov_b32_e32 v3, 0xcccccccc
-; GFX942-NEXT: v_mov_b32_e32 v4, 0xbbbbbbbb
-; GFX942-NEXT: v_mov_b32_e32 v5, 0xaaaaaaaa
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB8_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: ds_write_b128 v0, v[2:5]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_add_u32_e32 v0, 16, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB8_2
-; GFX942-NEXT: .LBB8_3: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i128_dynlen_lds:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB8_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xdddddddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0xcccccccc
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0xbbbbbbbb
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0xaaaaaaaa
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB8_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: ds_write_b128 v0, v[2:5]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 16, v0
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB8_2
+; GFX942-SDAG-NEXT: .LBB8_3: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i128_dynlen_lds:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB8_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s5, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s6, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s7, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB8_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-GISEL-NEXT: ds_write_b128 v0, v[2:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 16, v0
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB8_2
+; GFX942-GISEL-NEXT: .LBB8_3: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(3) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i32_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
-; GFX942-LABEL: memset_pattern_i32_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v10, -4, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-NEXT: v_and_b32_e32 v8, 3, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB9_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v4, 0xaabbccdd
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB9_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 4
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB9_2
-; GFX942-NEXT: .LBB9_3: ; %Flow6
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB9_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b64 v[2:3], 2, v[2:3]
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 0xaabbccdd
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB9_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB9_5
-; GFX942-NEXT: .LBB9_6: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i32_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB9_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0xaabbccdd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB9_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 4
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB9_2
+; GFX942-SDAG-NEXT: .LBB9_3: ; %Flow6
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB9_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xaabbccdd
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB9_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB9_5
+; GFX942-SDAG-NEXT: .LBB9_6: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i32_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 3, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB9_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xaabbccdd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: .LBB9_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 4
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, 64, v12
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB9_2
+; GFX942-GISEL-NEXT: .LBB9_3: ; %Flow6
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB9_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 2, v[2:3]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 4, v[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0xaabbccdd
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: .LBB9_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB9_5
+; GFX942-GISEL-NEXT: .LBB9_6: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i32 u0xaabbccdd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i32_dynval_dynlen(ptr addrspace(1) align 16 %a, i32 %val, i64 %len) {
-; GFX942-LABEL: memset_pattern_i32_dynval_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v8, v3
-; GFX942-NEXT: v_and_b32_e32 v10, -4, v8
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-NEXT: v_and_b32_e32 v6, 3, v8
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB10_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB10_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 4
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[2:5], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB10_2
-; GFX942-NEXT: .LBB10_3: ; %Flow6
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB10_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b64 v[4:5], 2, v[8:9]
-; GFX942-NEXT: v_and_b32_e32 v4, -16, v4
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[4:5]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB10_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB10_5
-; GFX942-NEXT: .LBB10_6: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i32_dynval_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -4, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 3, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB10_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB10_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 4
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[2:5], off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[12:13], 0, 64
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB10_2
+; GFX942-SDAG-NEXT: .LBB10_3: ; %Flow6
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB10_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[8:9]
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v4
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[4:5]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB10_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 4
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB10_5
+; GFX942-SDAG-NEXT: .LBB10_6: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i32_dynval_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, 3, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v8, v6
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v9, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB10_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: .LBB10_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 4
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[2:5], off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, 64, v12
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB10_2
+; GFX942-GISEL-NEXT: .LBB10_3: ; %Flow6
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB10_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[8:9]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 4, v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: .LBB10_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB10_5
+; GFX942-GISEL-NEXT: .LBB10_6: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i32 %val, i64 %len, i1 false)
ret void
}
@@ -503,152 +682,295 @@ define void @memset_pattern_i32_dynval_dynlen(ptr addrspace(1) align 16 %a, i32
; For i96, the store size and the alloc size differ on amdgpu, this case is not
; optimized.
define void @memset_pattern_i96_dynval_dynlen(ptr addrspace(1) align 16 %a, i96 %val, i64 %len) {
-; GFX942-LABEL: memset_pattern_i96_dynval_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-NEXT: v_mov_b32_e32 v6, v5
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB11_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB11_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: global_store_dwordx3 v[0:1], v[2:4], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB11_2
-; GFX942-NEXT: .LBB11_3: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i96_dynval_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v5
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB11_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB11_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: global_store_dwordx3 v[0:1], v[2:4], off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 16
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB11_2
+; GFX942-SDAG-NEXT: .LBB11_3: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i96_dynval_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB11_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: .LBB11_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: global_store_dwordx3 v[0:1], v[2:4], off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 16, v0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB11_2
+; GFX942-GISEL-NEXT: .LBB11_3: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i96 %val, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i32 %offset, i64 %len) {
-; GFX942-LABEL: memset_pattern_i64_as7_dynlen:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v8, v1
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_and_b32_e32 v10, -2, v8
-; GFX942-NEXT: v_and_b32_e32 v6, 1, v8
-; GFX942-NEXT: v_mov_b32_e32 v7, 0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v2, 0xccccdddd
-; GFX942-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
-; GFX942-NEXT: v_add_u32_e32 v1, s16, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB12_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 2
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
-; GFX942-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_2
-; GFX942-NEXT: .LBB12_3: ; %Flow3
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v8
-; GFX942-NEXT: v_and_b32_e32 v1, -16, v1
-; GFX942-NEXT: v_add3_u32 v2, v0, v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v0, 0xccccdddd
-; GFX942-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB12_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 1
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
-; GFX942-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v2, 8, v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_5
-; GFX942-NEXT: .LBB12_6: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -2, v8
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 1, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xccccdddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: .LBB12_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s8, s8, 2
+; GFX942-SDAG-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow3
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v8
+; GFX942-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
+; GFX942-SDAG-NEXT: v_add3_u32 v2, v0, v1, s16
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: .LBB12_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s8, s8, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
+; GFX942-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v2, 8, v2
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-SDAG-NEXT: .LBB12_6: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 1
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, 1, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v8, v6
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v9, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xccccdddd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xaaaabbbb
+; GFX942-GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: .LBB12_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s8, s8, 2
+; GFX942-GISEL-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
+; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow3
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 1, v[8:9]
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v2
+; GFX942-GISEL-NEXT: v_add3_u32 v2, v0, v1, s16
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: .LBB12_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s8, s8, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
+; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v2, 8, v2
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-GISEL-NEXT: .LBB12_6: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 u0xaaaabbbbccccdddd, i64 %len, i1 false)
ret void
}
define void @memset_pattern_i64_as7_dynlen_dynval(ptr addrspace(7) inreg align 16 %a, i32 %offset, i64 %len, i64 %val) {
-; GFX942-LABEL: memset_pattern_i64_as7_dynlen_dynval:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_and_b32_e32 v6, -2, v4
-; GFX942-NEXT: v_mov_b32_e32 v8, v3
-; GFX942-NEXT: v_and_b32_e32 v2, 1, v4
-; GFX942-NEXT: v_mov_b32_e32 v3, 0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_3
-; GFX942-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-NEXT: v_add_u32_e32 v1, s16, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v8
-; GFX942-NEXT: v_mov_b32_e32 v11, v9
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB13_2: ; %memset.pattern-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 2
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
-; GFX942-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_2
-; GFX942-NEXT: .LBB13_3: ; %Flow3
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_6
-; GFX942-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v4
-; GFX942-NEXT: v_and_b32_e32 v1, -16, v1
-; GFX942-NEXT: v_add3_u32 v0, v0, v1, s16
-; GFX942-NEXT: s_mov_b64 s[8:9], 0
-; GFX942-NEXT: .LBB13_5: ; %memset.pattern-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s8, s8, 1
-; GFX942-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[2:3]
-; GFX942-NEXT: buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
-; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_add_u32_e32 v0, 8, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_5
-; GFX942-NEXT: .LBB13_6: ; %Flow1
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen_dynval:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -2, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, 1, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v9
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: .LBB13_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s8, s8, 2
+; GFX942-SDAG-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow3
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
+; GFX942-SDAG-NEXT: v_add3_u32 v0, v0, v1, s16
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: .LBB13_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s8, s8, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[2:3]
+; GFX942-SDAG-NEXT: buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 8, v0
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-SDAG-NEXT: .LBB13_6: ; %Flow1
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen_dynval:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], 1
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 1, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], v[8:9]
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: .LBB13_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s8, s8, 2
+; GFX942-GISEL-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[4:5]
+; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-GISEL-NEXT: .LBB13_3: ; %Flow3
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[4:5], 1, v[6:7]
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v4
+; GFX942-GISEL-NEXT: v_add3_u32 v0, v0, v1, s16
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: .LBB13_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s8, s8, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[2:3]
+; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 8, v0
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-GISEL-NEXT: .LBB13_6: ; %Flow1
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 %val, i64 %len, i1 false)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/minimummaximum.ll b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
index bf82555031388..10f6984276d69 100644
--- a/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,TRUE16,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,TRUE16,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,TRUE16,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define amdgpu_ps float @test_minmax_f32(float %a, float %b, float %c) {
; GCN-LABEL: test_minmax_f32:
@@ -204,27 +204,48 @@ define amdgpu_ps half @test_maxmin_commuted_f16(half %a, half %b, half %c) {
}
define amdgpu_ps void @s_test_minmax_f16(half inreg %a, half inreg %b, half inreg %c, ptr addrspace(1) inreg %out) {
-; TRUE16-LABEL: s_test_minmax_f16:
-; TRUE16: ; %bb.0:
-; TRUE16-NEXT: v_maximum_f16 v0.l, s0, s1
-; TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; TRUE16-NEXT: s_mov_b32 s5, s4
-; TRUE16-NEXT: s_mov_b32 s4, s3
-; TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, s2
-; TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
-; TRUE16-NEXT: s_endpgm
+; GFX1170-SDAG-TRUE16-LABEL: s_test_minmax_f16:
+; GFX1170-SDAG-TRUE16: ; %bb.0:
+; GFX1170-SDAG-TRUE16-NEXT: v_maximum_f16 v0.l, s0, s1
+; GFX1170-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1170-SDAG-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX1170-SDAG-TRUE16-NEXT: s_mov_b32 s4, s3
+; GFX1170-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1170-SDAG-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, s2
+; GFX1170-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GFX1170-SDAG-TRUE16-NEXT: s_endpgm
;
-; FAKE16-LABEL: s_test_minmax_f16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: v_maximum_f16 v0, s0, s1
-; FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; FAKE16-NEXT: s_mov_b32 s5, s4
-; FAKE16-NEXT: s_mov_b32 s4, s3
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; FAKE16-NEXT: v_minimum_f16 v0, v0, s2
-; FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
-; FAKE16-NEXT: s_endpgm
+; GFX1170-SDAG-FAKE16-LABEL: s_test_minmax_f16:
+; GFX1170-SDAG-FAKE16: ; %bb.0:
+; GFX1170-SDAG-FAKE16-NEXT: v_maximum_f16 v0, s0, s1
+; GFX1170-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1170-SDAG-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX1170-SDAG-FAKE16-NEXT: s_mov_b32 s4, s3
+; GFX1170-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1170-SDAG-FAKE16-NEXT: v_minimum_f16 v0, v0, s2
+; GFX1170-SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GFX1170-SDAG-FAKE16-NEXT: s_endpgm
+;
+; GFX1170-GISEL-TRUE16-LABEL: s_test_minmax_f16:
+; GFX1170-GISEL-TRUE16: ; %bb.0:
+; GFX1170-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1170-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1170-GISEL-TRUE16-NEXT: s_mov_b32 s6, s3
+; GFX1170-GISEL-TRUE16-NEXT: s_mov_b32 s7, s4
+; GFX1170-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1170-GISEL-TRUE16-NEXT: v_maximumminimum_f16 v0.l, s0, s1, v0.l
+; GFX1170-GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
+; GFX1170-GISEL-TRUE16-NEXT: s_endpgm
+;
+; GFX1170-GISEL-FAKE16-LABEL: s_test_minmax_f16:
+; GFX1170-GISEL-FAKE16: ; %bb.0:
+; GFX1170-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
+; GFX1170-GISEL-FAKE16-NEXT: s_mov_b32 s6, s3
+; GFX1170-GISEL-FAKE16-NEXT: s_mov_b32 s7, s4
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_maximumminimum_f16 v0, s0, s1, v0
+; GFX1170-GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
+; GFX1170-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX12-SDAG-TRUE16-LABEL: s_test_minmax_f16:
; GFX12-SDAG-TRUE16: ; %bb.0:
@@ -253,23 +274,24 @@ define amdgpu_ps void @s_test_minmax_f16(half inreg %a, half inreg %b, half inre
; GFX12-GISEL-TRUE16: ; %bb.0:
; GFX12-GISEL-TRUE16-NEXT: s_maximum_f16 s0, s0, s1
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, s4
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, s3
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s6, s3
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s7, s4
; GFX12-GISEL-TRUE16-NEXT: s_minimum_f16 s0, s0, s2
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GFX12-GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
; GFX12-GISEL-FAKE16-LABEL: s_test_minmax_f16:
; GFX12-GISEL-FAKE16: ; %bb.0:
; GFX12-GISEL-FAKE16-NEXT: s_maximum_f16 s0, s0, s1
-; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s5, s4
-; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, s3
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s6, s3
+; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s7, s4
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-FAKE16-NEXT: s_minimum_f16 s0, s0, s2
-; GFX12-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[4:5]
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GFX12-GISEL-FAKE16-NEXT: s_endpgm
%smax = call half @llvm.maximum.f16(half %a, half %b)
%sminmax = call half @llvm.minimum.f16(half %smax, half %c)
@@ -281,8 +303,3 @@ declare half @llvm.minimum.f16(half, half)
declare half @llvm.maximum.f16(half, half)
declare float @llvm.minimum.f32(float, float)
declare float @llvm.maximum.f32(float, float)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1170-GISEL-FAKE16: {{.*}}
-; GFX1170-GISEL-TRUE16: {{.*}}
-; GFX1170-SDAG-FAKE16: {{.*}}
-; GFX1170-SDAG-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/minmax.ll b/llvm/test/CodeGen/AMDGPU/minmax.ll
index dba6532f06a02..03693c009f533 100644
--- a/llvm/test/CodeGen/AMDGPU/minmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/minmax.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,SDAG,SDAG-GFX11,SDAG-GFX11-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,SDAG,SDAG-GFX11,SDAG-GFX11-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GISEL,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1170,SDAG,SDAG-GFX1170,SDAG-GFX1170-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1170,SDAG,SDAG-GFX1170,SDAG-GFX1170-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1170,GISEL,GISEL-GFX1170,GISEL-GFX1170-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.70 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1170,GISEL,GISEL-GFX1170,GISEL-GFX1170-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1170,GISEL,GISEL-GFX1170,GISEL-GFX1170-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1170,GISEL,GISEL-GFX1170,GISEL-GFX1170-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,SDAG,SDAG-GFX12,SDAG-GFX12-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,SDAG,SDAG-GFX12,SDAG-GFX12-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-GFX12,GISEL-GFX12-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-GFX12,GISEL-GFX12-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-GFX12,GISEL-GFX12-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX12,GISEL,GISEL-GFX12,GISEL-GFX12-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1250,SDAG,SDAG-GFX1250,SDAG-GFX1250-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1250,SDAG,SDAG-GFX1250,SDAG-GFX1250-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1250,GISEL,GISEL-GFX1250,GISEL-GFX1250-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1250,GISEL,GISEL-GFX1250,GISEL-GFX1250-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1250,GISEL,GISEL-GFX1250,GISEL-GFX1250-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX1250,GISEL,GISEL-GFX1250,GISEL-GFX1250-FAKE16 %s
define i32 @test_minmax_i32(i32 %a, i32 %b, i32 %c) {
; GFX11-LABEL: test_minmax_i32:
@@ -1061,19 +1061,19 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
; GISEL-GFX11-TRUE16: ; %bb.0:
; GISEL-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s4, s3
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s7, s4
; GISEL-GFX11-TRUE16-NEXT: v_maxmin_f16 v0.l, s0, s1, v0.l
-; GISEL-GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX11-TRUE16-NEXT: s_endpgm
;
; GISEL-GFX11-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX11-FAKE16: ; %bb.0:
; GISEL-GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
-; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s4, s3
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s7, s4
; GISEL-GFX11-FAKE16-NEXT: v_maxmin_f16 v0, s0, s1, v0
-; GISEL-GFX11-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX11-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX11-FAKE16-NEXT: s_endpgm
;
; SDAG-GFX1170-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
@@ -1097,21 +1097,24 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
;
; GISEL-GFX1170-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX1170-TRUE16: ; %bb.0:
-; GISEL-GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GISEL-GFX1170-TRUE16-NEXT: s_max_f16 s0, s0, s1
; GISEL-GFX1170-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX1170-TRUE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX1170-TRUE16-NEXT: s_mov_b32 s4, s3
-; GISEL-GFX1170-TRUE16-NEXT: v_maxmin_num_f16 v0.l, s0, s1, v0.l
-; GISEL-GFX1170-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX1170-TRUE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX1170-TRUE16-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX1170-TRUE16-NEXT: s_min_f16 s0, s0, s2
+; GISEL-GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX1170-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX1170-TRUE16-NEXT: s_endpgm
;
; GISEL-GFX1170-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX1170-FAKE16: ; %bb.0:
-; GISEL-GFX1170-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
-; GISEL-GFX1170-FAKE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX1170-FAKE16-NEXT: s_mov_b32 s4, s3
-; GISEL-GFX1170-FAKE16-NEXT: v_maxmin_num_f16 v0, s0, s1, v0
-; GISEL-GFX1170-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX1170-FAKE16-NEXT: s_max_f16 s0, s0, s1
+; GISEL-GFX1170-FAKE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX1170-FAKE16-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX1170-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX1170-FAKE16-NEXT: s_min_f16 s0, s0, s2
+; GISEL-GFX1170-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX1170-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX1170-FAKE16-NEXT: s_endpgm
;
; SDAG-GFX12-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
@@ -1135,21 +1138,24 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
;
; GISEL-GFX12-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX12-TRUE16: ; %bb.0:
-; GISEL-GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GISEL-GFX12-TRUE16-NEXT: s_max_num_f16 s0, s0, s1
; GISEL-GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s4, s3
-; GISEL-GFX12-TRUE16-NEXT: v_maxmin_num_f16 v0.l, s0, s1, v0.l
-; GISEL-GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX12-TRUE16-NEXT: s_min_num_f16 s0, s0, s2
+; GISEL-GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX12-TRUE16-NEXT: s_endpgm
;
; GISEL-GFX12-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX12-FAKE16: ; %bb.0:
-; GISEL-GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
-; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s4, s3
-; GISEL-GFX12-FAKE16-NEXT: v_maxmin_num_f16 v0, s0, s1, v0
-; GISEL-GFX12-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX12-FAKE16-NEXT: s_max_num_f16 s0, s0, s1
+; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX12-FAKE16-NEXT: s_min_num_f16 s0, s0, s2
+; GISEL-GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX12-FAKE16-NEXT: s_endpgm
;
; SDAG-GFX1250-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
@@ -1182,12 +1188,13 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
; GISEL-GFX1250-TRUE16-NEXT: global_wb
; GISEL-GFX1250-TRUE16-NEXT: v_nop
; GISEL-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GISEL-GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GISEL-GFX1250-TRUE16-NEXT: s_max_num_f16 s0, s0, s1
; GISEL-GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX1250-TRUE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX1250-TRUE16-NEXT: s_mov_b32 s4, s3
-; GISEL-GFX1250-TRUE16-NEXT: v_maxmin_num_f16 v0.l, s0, s1, v0.l
-; GISEL-GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX1250-TRUE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX1250-TRUE16-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX1250-TRUE16-NEXT: s_min_num_f16 s0, s0, s2
+; GISEL-GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX1250-TRUE16-NEXT: s_endpgm
;
; GISEL-GFX1250-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
@@ -1195,11 +1202,13 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
; GISEL-GFX1250-FAKE16-NEXT: global_wb
; GISEL-GFX1250-FAKE16-NEXT: v_nop
; GISEL-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GISEL-GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
-; GISEL-GFX1250-FAKE16-NEXT: s_mov_b32 s5, s4
-; GISEL-GFX1250-FAKE16-NEXT: s_mov_b32 s4, s3
-; GISEL-GFX1250-FAKE16-NEXT: v_maxmin_num_f16 v0, s0, s1, v0
-; GISEL-GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GISEL-GFX1250-FAKE16-NEXT: s_max_num_f16 s0, s0, s1
+; GISEL-GFX1250-FAKE16-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX1250-FAKE16-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX1250-FAKE16-NEXT: s_min_num_f16 s0, s0, s2
+; GISEL-GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GISEL-GFX1250-FAKE16-NEXT: s_endpgm
%smax = call half @llvm.maxnum.f16(half %a, half %b)
%sminmax = call half @llvm.minnum.f16(half %smax, half %c)
@@ -1238,8 +1247,8 @@ define half @test_minmax_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX11-FAKE16-LABEL: test_minmax_commuted_f16_ieee_true:
; GISEL-GFX11-FAKE16: ; %bb.0:
; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GISEL-GFX11-FAKE16-NEXT: v_maxmin_f16 v0, v0, v1, v2
; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1274,8 +1283,8 @@ define half @test_minmax_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX1170-FAKE16-LABEL: test_minmax_commuted_f16_ieee_true:
; GISEL-GFX1170-FAKE16: ; %bb.0:
; GISEL-GFX1170-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GISEL-GFX1170-FAKE16-NEXT: v_maxmin_num_f16 v0, v0, v1, v2
; GISEL-GFX1170-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1326,8 +1335,8 @@ define half @test_minmax_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GISEL-GFX12-FAKE16-NEXT: v_maxmin_num_f16 v0, v0, v1, v2
; GISEL-GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1366,8 +1375,8 @@ define half @test_minmax_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX1250-FAKE16: ; %bb.0:
; GISEL-GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GISEL-GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GISEL-GFX1250-FAKE16-NEXT: v_maxmin_num_f16 v0, v0, v1, v2
; GISEL-GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
@@ -1504,8 +1513,8 @@ define half @test_maxmin_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX11-FAKE16-LABEL: test_maxmin_commuted_f16_ieee_true:
; GISEL-GFX11-FAKE16: ; %bb.0:
; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
; GISEL-GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GISEL-GFX11-FAKE16-NEXT: v_minmax_f16 v0, v0, v1, v2
; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1540,8 +1549,8 @@ define half @test_maxmin_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX1170-FAKE16-LABEL: test_maxmin_commuted_f16_ieee_true:
; GISEL-GFX1170-FAKE16: ; %bb.0:
; GISEL-GFX1170-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1170-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GISEL-GFX1170-FAKE16-NEXT: v_minmax_num_f16 v0, v0, v1, v2
; GISEL-GFX1170-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1592,8 +1601,8 @@ define half @test_maxmin_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GISEL-GFX12-FAKE16-NEXT: v_minmax_num_f16 v0, v0, v1, v2
; GISEL-GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1632,8 +1641,8 @@ define half @test_maxmin_commuted_f16_ieee_true(half %a, half %b, half %c) {
; GISEL-GFX1250-FAKE16: ; %bb.0:
; GISEL-GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GISEL-GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
; GISEL-GFX1250-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GISEL-GFX1250-FAKE16-NEXT: v_minmax_num_f16 v0, v0, v1, v2
; GISEL-GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index aa80c946124a4..c0e41672ae50f 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -49,13 +49,18 @@ define amdgpu_cs float @v_s_exp_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_exp_f16(half inreg %src) {
-; GFX12-LABEL: v_s_exp_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_exp_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_exp_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_exp_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: v_s_exp_f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_exp_f16_e32 v0.l, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_exp_f16:
; GCN-GISEL: ; %bb.0:
@@ -132,13 +137,18 @@ define amdgpu_cs float @v_s_log_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_log_f16(half inreg %src) {
-; GFX12-LABEL: v_s_log_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_log_f16 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_log_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: v_s_log_f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_log_f16_e32 v0.l, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_log_f16:
; GCN-GISEL: ; %bb.0:
@@ -625,13 +635,18 @@ define amdgpu_cs float @srcmods_neg_f32(float inreg %src) {
}
define amdgpu_cs half @srcmods_abs_f16(half inreg %src) {
-; GFX12-LABEL: srcmods_abs_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_log_f16 s0, |s0|
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: srcmods_abs_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, |s0|
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: srcmods_abs_f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_log_f16_e64 v0.l, |s0|
+; GFX12-GISEL-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: srcmods_abs_f16:
; GCN-GISEL: ; %bb.0:
@@ -643,13 +658,18 @@ define amdgpu_cs half @srcmods_abs_f16(half inreg %src) {
}
define amdgpu_cs half @srcmods_neg_f16(half inreg %src) {
-; GFX12-LABEL: srcmods_neg_f16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_s_log_f16 s0, -s0
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: srcmods_neg_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, -s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: srcmods_neg_f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_log_f16_e64 v0.l, -s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: srcmods_neg_f16:
; GCN-GISEL: ; %bb.0:
@@ -684,13 +704,26 @@ define amdgpu_cs float @fdiv_f32_i32(float inreg %a, i32 inreg %b) {
}
define amdgpu_cs half @fdiv_f16_i16(half inreg %a, i16 inreg %b) {
-; GFX12-LABEL: fdiv_f16_i16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_cvt_f16_u16_e32 v0.l, s1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX12-NEXT: v_rcp_f16_e32 v0.l, v0.l
-; GFX12-NEXT: v_mul_f16_e32 v0.l, s0, v0.l
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: fdiv_f16_i16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_cvt_f16_u16_e32 v0.l, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_rcp_f16_e32 v0.l, v0.l
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.l, s0, v0.l
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: fdiv_f16_i16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_cvt_f16_u16_e32 v0.l, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_s_rcp_f16 s1, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT: s_mul_f16 s0, s0, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: fdiv_f16_i16:
; GCN-GISEL: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
index d0b123172c698..ef7e06883a84f 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
@@ -184,12 +184,22 @@ define amdgpu_vs half @ftrunc_f16(half inreg %val) {
}
define amdgpu_vs half @frint_f16(half inreg %val) {
-; CHECK-LABEL: frint_f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_rndne_f16 s0, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-LABEL: frint_f16:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_rndne_f16 s0, s0
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: frint_f16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: v_rndne_f16_e32 v0.l, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: frint_f16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: v_rndne_f16_e32 v0.l, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.rint.f16(half %val)
ret half %res
}
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
index 893b6f0cd4f51..a0f3178fae5f7 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.50 < %s | FileCheck -check-prefixes=CHECK,GFX1150 %s
-; RUN: llc -mtriple=amdgpu11.50 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=CHECK,GFX1150 %s
+; RUN: llc -mtriple=amdgpu11.50 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX1150 %s
; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=CHECK,GFX12 %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=CHECK,GFX12 %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX12 %s
define amdgpu_vs float @fadd_f32(float inreg %a, float inreg %b) {
; CHECK-LABEL: fadd_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
index 198188d1618dd..9185658b594c0 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GCN,SDAG,GCN-FAKE16,SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GCN,GISEL,GCN-FAKE16,GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GCN,GISEL,GCN-FAKE16,GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GCN,SDAG,GCN-REAL16,SDAG-REAL16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GCN,GISEL,GCN-REAL16,GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GCN,GISEL,GCN-REAL16,GISEL-REAL16 %s
define amdgpu_ps float @flat_load_b32_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b32_idxprom:
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
index 88f1442a08f9d..36ec67b0b649b 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX1250,GFX1250-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu13.10 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX13,GFX13-SDAG,GFX13-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu13.10 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX13,GFX13-GISEL,GFX13-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu13.10 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX13,GFX13-GISEL,GFX13-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu13.10 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX13,GFX13-SDAG,GFX13-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu13.10 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX13,GFX13-GISEL,GFX13-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu13.10 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX13,GFX13-GISEL,GFX13-TRUE16 %s
define amdgpu_ps float @global_load_b32_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
index 5634b33f57b0c..660252a9f01bf 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -mtriple=amdgpu12.50 < %s | FileCheck --check-prefixes=GCN,SDAG %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck --check-prefixes=GCN,GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu12.50 < %s | FileCheck --check-prefixes=GCN,GISEL %s
define amdgpu_ps float @s_load_b32_idxprom(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b32_idxprom:
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
index 2c4d120b6b9a2..ba91e9230d81f 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
; FIXME: promotion not handled without f16 insts
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
index 7473ddba61662..cc8abe6535b05 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
; FIXME: promotion not handled without f16 insts
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 32f014bfdda57..5b61d49c7db5b 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
; FIXME: promotion not handled without f16 insts
diff --git a/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
index 166ec22538553..110408d8943f8 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
@@ -9,7 +9,7 @@
; XUN: llc -global-isel=1 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; define half @test_ldexp_f16_i16(ptr addrspace(1) %out, half %a, i16 %b) #0 {
@@ -18,23 +18,23 @@
; }
define half @test_ldexp_f16_i32(ptr addrspace(1) %out, half %a, i32 %b) #0 {
-; GFX8-LABEL: test_ldexp_f16_i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX8-NEXT: v_med3_i32 v0, v3, s4, v0
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v2, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_ldexp_f16_i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v0, v3, s4, v0
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_f16_i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX9-NEXT: v_med3_i32 v0, v3, s4, v0
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v2, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_f16_i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v0, v3, s4, v0
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_f16_i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -54,6 +54,24 @@ define half @test_ldexp_f16_i32(ptr addrspace(1) %out, half %a, i32 %b) #0 {
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX8-GISEL-LABEL: test_ldexp_f16_i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v0, v3, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_ldexp_f16_i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v0, v3, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_f16_i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -66,9 +84,9 @@ define half @test_ldexp_f16_i32(ptr addrspace(1) %out, half %a, i32 %b) #0 {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_f16_i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, v3, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, 0xffff8000, v3, v0
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.experimental.constrained.ldexp.f16.i32(half %a, i32 %b, metadata !"round.dynamic", metadata !"fpexcept.strict")
@@ -372,21 +390,21 @@ define <4 x half> @test_ldexp_v4f16_v4i32(ptr addrspace(1) %out, <4 x half> %a,
}
define amdgpu_ps half @s_test_ldexp_f16_i32(half inreg %a, i32 inreg %b) #0 {
-; GFX8-LABEL: s_test_ldexp_f16_i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX8-NEXT: v_med3_i32 v0, s1, v0, v1
-; GFX8-NEXT: v_ldexp_f16_e32 v0, s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: s_test_ldexp_f16_i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v0, s1, v0, v1
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, s0, v0
+; GFX8-SDAG-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_test_ldexp_f16_i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX9-NEXT: v_med3_i32 v0, s1, v0, v1
-; GFX9-NEXT: v_ldexp_f16_e32 v0, s0, v0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: s_test_ldexp_f16_i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v0, s1, v0, v1
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, s0, v0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-TRUE16-LABEL: s_test_ldexp_f16_i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -404,20 +422,36 @@ define amdgpu_ps half @s_test_ldexp_f16_i32(half inreg %a, i32 inreg %b) #0 {
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v0, s0, v0
; GFX11-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
+; GFX8-GISEL-LABEL: s_test_ldexp_f16_i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_max_i32 s1, s1, 0xffff8000
+; GFX8-GISEL-NEXT: s_min_i32 s1, s1, 0x7fff
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: s_test_ldexp_f16_i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_max_i32 s1, s1, 0xffff8000
+; GFX9-GISEL-NEXT: s_min_i32 s1, s1, 0x7fff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, s0, v0
+; GFX9-GISEL-NEXT: ; return to shader part epilog
+;
; GFX11-GISEL-TRUE16-LABEL: s_test_ldexp_f16_i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_med3_i32 v0, 0xffff8000, s1, v0
-; GFX11-GISEL-TRUE16-NEXT: v_ldexp_f16_e32 v0.l, s0, v0.l
+; GFX11-GISEL-TRUE16-NEXT: s_max_i32 s1, s1, 0xffff8000
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT: s_min_i32 s1, s1, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_ldexp_f16_e64 v0.l, s0, s1
; GFX11-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-FAKE16-LABEL: s_test_ldexp_f16_i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, 0xffff8000, s1, v0
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, s0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_max_i32 s1, s1, 0xffff8000
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: s_min_i32 s1, s1, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e64 v0, s0, s1
; GFX11-GISEL-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.experimental.constrained.ldexp.f16.i32(half %a, i32 %b, metadata !"round.dynamic", metadata !"fpexcept.strict")
ret half %result
@@ -437,7 +471,3 @@ attributes #1 = { nocallback nofree nosync nounwind willreturn memory(inaccessib
; GFX11: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
-; GFX8-GISEL: {{.*}}
-; GFX8-SDAG: {{.*}}
-; GFX9-GISEL: {{.*}}
-; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/v_pack.ll b/llvm/test/CodeGen/AMDGPU/v_pack.ll
index 9cd496fc80ff3..91e1849239b1f 100644
--- a/llvm/test/CodeGen/AMDGPU/v_pack.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_pack.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu10.10 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu10.10 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GISEL %s
+; RUN: llc -global-isel -amdgpu-scalarize-global-loads=false -mtriple=amdgpu10.10 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GISEL %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-GCN-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-GISEL-FAKE16 %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-GCN-REAL16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-GISEL-REAL16 %s
+; RUN: llc -global-isel -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11-GISEL-REAL16 %s
declare i32 @llvm.amdgcn.workitem.id.x() #1
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index 38f42c6e4be6a..1fdd038aca751 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -8,9 +8,9 @@
; RUN: llc -mtriple=amdgpu8.03 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GISEL-VI %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GISEL-GFX9 %s
-; RUN: llc -mtriple=amdgpu11.01 -mattr=+real-true16 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgpu11.01 -mattr=+real-true16 -global-isel < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.01 -mattr=-real-true16 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
-; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GISEL-GFX12,GISEL-GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 -global-isel < %s | FileCheck -check-prefixes=GISEL-GFX12,GISEL-GFX12-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GISEL-GFX12,GISEL-GFX12-FAKE16 %s
; <GFX9 has no V_SAT_PK, GFX9+ has V_SAT_PK, GFX11 has V_SAT_PK with t16
More information about the llvm-commits
mailing list